Replace protobufs with flatbuffers

.gitignore

@@ -6,11 +6,10 @@ node_modules
 
 # git deps
 /third_party/v8/
-/third_party/tools/protoc_wrapper/
 /third_party/cpplint/
-/third_party/protobuf/
 /third_party/zlib/
 /third_party/rust_crates/libc/
+/third_party/flatbuffers/src/
 
 # gclient files
 /third_party/.gclient_entries

BUILD.gn

@@ -1,6 +1,7 @@
-import("//third_party/protobuf/proto_library.gni")
 import("//third_party/v8/gni/v8.gni")
 import("//third_party/v8/snapshot_toolchain.gni")
+import("//third_party/flatbuffers/flatbuffer.gni")
+import("//third_party/flatbuffers/ts_flatbuffer.gni")
 import("deno.gni")
 import("rust.gni")
 
@@ -30,9 +31,9 @@ executable("mock_main") {
  "src/mock_main.cc",
  ]
  deps = [
+ ":flatbufferjs",
  ":libdeno",
- ":msg_proto",
- "//third_party/protobuf:protoc_lib",
+ ":msg_cpp",
  ]
  configs += [ ":deno_config" ]
 }
@@ -89,26 +90,24 @@ executable("snapshot_creator") {
  configs += [ ":deno_config" ]
 }
 
-proto_library("msg_proto") {
+flatbuffer("msg_cpp") {
  sources = [
- "src/msg.proto",
+ "src/msg.fbs",
  ]
- generate_python = false
 }
 
 run_node("bundle") {
  out_dir = "$target_gen_dir/bundle/"
  sources = [
  "js/main.ts",
- "js/msg.pb.d.ts",
- "js/msg.pb.js",
+ "js/msg_generated.ts",
  "js/package.json", # The `browserslist` field controls Babel behavior.
  ]
  outputs = [
  out_dir + "main.js",
  ]
  deps = [
- ":protobufjs",
+ ":flatbufferjs",
  ":run_tsc",
  ]
  args = [
@@ -128,16 +127,15 @@ run_node("run_tsc") {
  tsconfig = "tsconfig.json"
  out_dir = "$target_gen_dir/tsc_dist/"
  sources = [
- "js/msg.pb.d.ts",
- "js/msg.pb.js",
+ "js/msg_generated.ts",
  main,
  tsconfig,
  ]
  outputs = [
  out_dir + "/main.js",
  ]
  deps = [
- ":protobufjs",
+ ":flatbufferjs",
  ]
  args = [
  "./node_modules/typescript/bin/tsc",
@@ -148,26 +146,38 @@ run_node("run_tsc") {
  ]
 }
 
-# Generates protobufjs code.
-# TODO(ry) Ideally protobufjs output files should be written into
+# Generates flatbuffer TypeScript code.
+# TODO(ry) Ideally flatc output files should be written into
 # target_gen_dir, but its difficult to get this working in a way that the
 # bundler can resolve their location. (The bundler does not support NODE_PATH?)
-# Therefore this hack: write the generated msg.pb.js and msg.pb.d.ts outputs
-# into the js/ folder, and we check them into the repo. Hopefully this hack can
-# be removed at some point. If msg.proto is changed, commit changes to the
+# Therefore this hack: write the msg_generated.ts output
+# into the js/ folder, and we check it into the repo. Hopefully this hack can
+# be removed at some point. If msg.fps is changed, commit changes to the
 # generated JS files. The stamp file is just to make gn work.
-action("protobufjs") {
- script = "js/pbjs_hack.py"
+action("flatbufferjs") {
+ script = "js/flatbufferjs_hack.py"
  sources = [
- "src/msg.proto",
+ "src/msg.fbs",
  ]
  outputs = [
- "$target_gen_dir/pbjs_hack.stamp",
+ "$target_gen_dir/flatbufferjs_hack.stamp",
  ]
+
  args = [
- rebase_path(sources[0], root_build_dir),
+ rebase_path("$target_gen_dir/msg_generated.ts", root_build_dir),
+ rebase_path("js/msg_generated.ts", root_build_dir),
  rebase_path(outputs[0], root_build_dir),
  ]
+
+ deps = [
+ ":msg_ts",
+ ]
+}
+
+ts_flatbuffer("msg_ts") {
+ sources = [
+ "src/msg.fbs",
+ ]
 }
 
 # Generates $target_gen_dir/snapshot_deno.cc

gclient_config.py

@@ -17,14 +17,6 @@
  'v8/test/test262/harness': None,
  'v8/tools/luci-go': None
  }
-}, {
- 'url': 'https://github.com/ry/protobuf_chromium.git',
- 'name': 'protobuf',
-}, {
- 'url':
- 'https://chromium.googlesource.com/chromium/src/tools/protoc_wrapper@9af82fef8cb9ca3ccc13e2ed958f58f2c21f449b',
- 'name':
- 'tools/protoc_wrapper'
 }, {
  'url':
  'https://chromium.googlesource.com/chromium/src/third_party/zlib@39b4a6260702da4c089eca57136abf40a39667e9',
@@ -35,6 +27,13 @@
  'https://github.com/cpplint/cpplint.git@a33992f68f36fcaa6d0f531a25012a4c474d3542',
  'name':
  'cpplint'
+}, {
+ # Tracking a bleeding-edge branch that is nearing rust support.
+ # https://github.com/google/flatbuffers/pull/3894
+ 'url':
+ 'https://github.com/rw/flatbuffers.git@2018-02--rust',
+ 'name':
+ 'flatbuffers/src'
 }, {
  'url':
  'https://github.com/rust-lang/libc.git@8a85d662b90c14d458bc4ae9521a05564e20d7ae',

js/flatbufferjs_hack.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+"""
+gn can only run python scripts.
+
+Generates flatbuffer TypeScript code.
+"""
+import subprocess
+import sys
+import os
+import shutil
+
+# TODO(ry) Ideally flatc output files should be written into target_gen_dir, but
+# its difficult to get this working in a way that parcel can resolve their
+# location. (Parcel does not support NODE_PATH.) Therefore this hack: write the
+# generated msg_generated.ts outputs into the js/ folder, and we check them into
+# the repo. Hopefully this hack can be removed at some point. If msg.fps is
+# changed, commit changes to the generated JS file.
+
+src = sys.argv[1]
+dst = sys.argv[2]
+stamp_file = sys.argv[3]
+
+shutil.copyfile(src, dst)
+
+
+def touch(fname):
+ if os.path.exists(fname):
+ os.utime(fname, None)
+ else:
+ open(fname, 'a').close()
+
+
+touch(stamp_file)

js/main.ts

@@ -1,7 +1,9 @@
 // tslint:disable-next-line:no-reference
 /// <reference path="deno.d.ts" />
 import * as ts from "typescript";
-import { deno as pb } from "./msg.pb";
+
+import { flatbuffers } from "flatbuffers";
+import { deno as fbs } from "./msg_generated";
 
 const globalEval = eval;
 const window = globalEval("this");
@@ -11,22 +13,20 @@ window["denoMain"] = () => {
  const res = deno.send("startDeno2", emptyArrayBuffer());
  // deno.print(`after`);
  const resUi8 = new Uint8Array(res);
- deno.print(`before`);
- const msg = pb.Msg.decode(resUi8);
- deno.print(`after`);
- const {
- startCwd: cwd,
- startArgv: argv,
- startDebugFlag: debugFlag,
- startMainJs: mainJs,
- startMainMap: mainMap
- } = msg;
 
+ const bb = new flatbuffers.ByteBuffer(resUi8);
+ const msg = fbs.Msg.getRootAsMsg(bb);
+
+ // startDebugFlag: debugFlag,
+ // startMainJs: mainJs,
+ // startMainMap: mainMap
+ const cwd = msg.startCwd();
  deno.print(`cwd: ${cwd}`);
- deno.print(`debugFlag: ${debugFlag}`);
 
- for (let i = 0; i < argv.length; i++) {
- deno.print(`argv[${i}] ${argv[i]}`);
+ const argv: string[] = [];
+ for (let i = 0; i < msg.startArgvLength(); i++) {
+ const arg = msg.startArgv(i);
+ deno.print(`argv[${i}] ${arg}`);
  }
 };