Clean up tools/

- Factor out tools/util.py
- Move js/*.py to tools.
- Rewrite tools/format.sh in python.
- Run lint first in travis.

.gitignore

@@ -1,5 +1,6 @@
 # build
 /out/
+*.pyc
 
 # npm deps
 node_modules

.travis.yml

@@ -44,8 +44,8 @@ install:
  # Travis hangs without -j2 argument to ninja.
  - ninja -j2 -C $BUILD_PATH mock_runtime_test handlers_test deno_cc deno
 script:
+ - ./tools/lint.py
  - $BUILD_PATH/mock_runtime_test
  - $BUILD_PATH/handlers_test
  - $BUILD_PATH/deno_cc foo bar
  - $BUILD_PATH/deno meow
- - ./tools/lint.py

BUILD.gn

@@ -177,7 +177,7 @@ run_node("run_tsc") {
 # be removed at some point. If msg.fps is changed, commit changes to the
 # generated JS files. The stamp file is just to make gn work.
 action("flatbufferjs") {
- script = "js/flatbufferjs_hack.py"
+ script = "//tools/flatbufferjs_hack.py"
  sources = [
  "src/msg.fbs",
  ]

build_extra/deno.gni

@@ -1,7 +1,7 @@
 template("run_node") {
  action(target_name) {
  forward_variables_from(invoker, "*")
- script = "//js/run_node.py"
+ script = "//tools/run_node.py"
  }
 }
 

build_extra/flatbuffers/BUILD.gn

@@ -100,7 +100,6 @@ flatbuffer("flatbuffers_samplebuffer") {
  ]
  flatc_include_dirs = [ "$fb_src/tests/include_test" ]
 }
-
 # test("flatbuffers_unittest") {
 # sources = [
 # "src/tests/test.cpp",

build_extra/flatbuffers/flatbuffer.gni

@@ -188,15 +188,21 @@ template("ts_flatbuffer") {
  copy_name = target_name + "_copy"
 
  copy(copy_name) {
- sources = [ "$flatbuffers_source_location/js/flatbuffers.js" ]
- outputs = [ "$target_gen_dir/flatbuffers.js" ]
+ sources = [
+ "$flatbuffers_source_location/js/flatbuffers.js",
+ ]
+ outputs = [
+ "$target_gen_dir/flatbuffers.js",
+ ]
  }
 
  compiled_action_foreach(target_name) {
  tool = "$flatbuffers_build_location:flatc"
 
  sources = invoker.sources
- deps = [ ":" + copy_name ]
+ deps = [
+ ":" + copy_name,
+ ]
 
  out_dir = target_gen_dir
 

tools/build_third_party.py

@@ -10,58 +10,22 @@
 import os
 from os.path import join
 import subprocess
+from util import run, remove_and_symlink
 
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 third_party_path = join(root_path, "third_party")
 
-
-def main():
- try:
- os.makedirs(third_party_path)
- except:
- pass
- os.chdir(third_party_path)
- remove_and_symlink(join("..", "gclient_config.py"), ".gclient")
- remove_and_symlink(join("..", "package.json"), "package.json")
- remove_and_symlink(join("..", "yarn.lock"), "yarn.lock")
- remove_and_symlink(join("v8", "third_party", "googletest"), "googletest")
- remove_and_symlink(join("v8", "third_party", "jinja2"), "jinja2")
- remove_and_symlink(join("v8", "third_party", "llvm-build"), "llvm-build")
- remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe")
- run(["gclient", "sync", "--no-history"])
- run(["yarn"])
-
-
-def run(args):
- print " ".join(args)
- env = os.environ.copy()
- subprocess.check_call(args, env=env)
-
-
-def remove_and_symlink(target, name):
- try:
- os.unlink(name)
- except:
- pass
- os.symlink(target, name)
-
-
-def symlink(target, name, target_is_dir=False):
- if os.name == "nt":
- import ctypes
- CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW
- CreateSymbolicLinkW.restype = ctypes.c_ubyte
- CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p,
- ctypes.c_uint32)
-
- flags = 0x02 # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
- if (target_is_dir):
- flags |= 0x01 # SYMBOLIC_LINK_FLAG_DIRECTORY
- if not CreateSymbolicLinkW(name, target, flags):
- raise ctypes.WinError()
- else:
- os.symlink(target, name)
-
-
-if '__main__' == __name__:
- main()
+try:
+ os.makedirs(third_party_path)
+except:
+ pass
+os.chdir(third_party_path)
+remove_and_symlink(join("..", "gclient_config.py"), ".gclient")
+remove_and_symlink(join("..", "package.json"), "package.json")
+remove_and_symlink(join("..", "yarn.lock"), "yarn.lock")
+remove_and_symlink(join("v8", "third_party", "googletest"), "googletest")
+remove_and_symlink(join("v8", "third_party", "jinja2"), "jinja2")
+remove_and_symlink(join("v8", "third_party", "llvm-build"), "llvm-build")
+remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe")
+run(["gclient", "sync", "--no-history"])
+run(["yarn"])

js/flatbufferjs_hack.py → tools/flatbufferjs_hack.py

@@ -8,6 +8,7 @@
 import sys
 import os
 import shutil
+import util
 
 # TODO(ry) Ideally flatc output files should be written into target_gen_dir, but
 # its difficult to get this working in a way that parcel can resolve their
@@ -22,12 +23,4 @@
 
 shutil.copyfile(src, dst)
 
-
-def touch(fname):
- if os.path.exists(fname):
- os.utime(fname, None)
- else:
- open(fname, 'a').close()
-
-
-touch(stamp_file)
+util.touch(stamp_file)

tools/format.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+import os
+from glob import glob
+from util import run
+
+root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+os.chdir(root_path)
+# TODO(ry) Install clang-format in third_party.
+run(["clang-format", "-i", "-style", "Google"] + glob("src/*.cc") +
+ glob("src/*.h"))
+for fn in ["BUILD.gn", ".gn"] + glob("build_extra/**/*.gn*"):
+ run(["gn", "format", fn])
+# TODO(ry) Install yapf in third_party.
+run(["yapf", "-i"] + glob("tools/*.py"))
+# TODO(ry) Install prettier in third_party.
+run([
+ "prettier", "--write", "js/deno.d.ts", "js/main.ts", "js/mock_runtime.js",
+ "tsconfig.json"
+])
+# Do not format these.
+# js/msg_generated.ts
+# js/flatbuffers.js
+run(["rustfmt", "--write-mode", "overwrite"] + glob("src/*.rs"))

tools/lint.py

@@ -2,29 +2,17 @@
 # Does google-lint on c++ files and ts-lint on typescript files
 
 import os
-import subprocess
+from util import run
 
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 third_party_path = os.path.join(root_path, "third_party")
 cpplint = os.path.join(third_party_path, "cpplint", "cpplint.py")
 tslint = os.path.join(third_party_path, "node_modules", "tslint", "bin",
  "tslint")
 
-
-def run(args):
- print(" ".join(args))
- env = os.environ.copy()
- subprocess.check_call(args, env=env)
-
-
-def main():
- os.chdir(root_path)
- run([
- "python", cpplint, "--filter=-build/include_subdir",
- "--repository=src", "--extensions=cc,h", "--recursive", "src/."
- ])
- run(["node", tslint, "-p", ".", "--exclude", "js/msg_generated.ts"])
-
-
-if __name__ == "__main__":
- main()
+os.chdir(root_path)
+run([
+ "python", cpplint, "--filter=-build/include_subdir", "--repository=src",
+ "--extensions=cc,h", "--recursive", "src/."
+])
+run(["node", tslint, "-p", ".", "--exclude", "js/msg_generated.ts"])

tools/run_node.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+gn can only run python scripts. This launches a subprocess Node process.
+The working dir of this program is out/Debug/ (AKA root_build_dir)
+Before running node, we symlink js/node_modules to out/Debug/node_modules.
+"""
+import subprocess
+import sys
+import os
+import util
+
+root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+tools_path = os.path.join(root_path, "tools")
+third_party_path = os.path.join(root_path, "third_party")
+target_abs = os.path.join(third_party_path, "node_modules")
+target_rel = os.path.relpath(target_abs)
+
+util.remove_and_symlink(target_rel, "node_modules", True)
+util.run(["node"] + sys.argv[1:])

tools/util.py

@@ -0,0 +1,42 @@
+# Copyright 2018 Ryan Dahl <[email protected]>
+# All rights reserved. MIT License.
+import os
+import subprocess
+
+
+def run(args):
+ print " ".join(args)
+ env = os.environ.copy()
+ subprocess.check_call(args, env=env)
+
+
+def remove_and_symlink(target, name, target_is_dir=False):
+ try:
+ os.unlink(name)
+ except:
+ pass
+ symlink(target, name, target_is_dir)
+
+
+def symlink(target, name, target_is_dir=False):
+ if os.name == "nt":
+ import ctypes
+ CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW
+ CreateSymbolicLinkW.restype = ctypes.c_ubyte
+ CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p,
+ ctypes.c_uint32)
+
+ flags = 0x02 # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
+ if (target_is_dir):
+ flags |= 0x01 # SYMBOLIC_LINK_FLAG_DIRECTORY
+ if not CreateSymbolicLinkW(name, target, flags):
+ raise ctypes.WinError()
+ else:
+ os.symlink(target, name)
+
+
+def touch(fname):
+ if os.path.exists(fname):
+ os.utime(fname, None)
+ else:
+ open(fname, 'a').close()