Add benchmark tools (denoland#777)

.gitignore

@@ -6,7 +6,11 @@ Cargo.lock
 yarn.lock
 # npm deps
 node_modules
+# editor files
 .idea
-
 # RLS generated files
 /target/
+# export dir for gh-pages
+/gh-pages
+# temp benchmark data
+/website/data.json

.travis.yml

@@ -37,11 +37,24 @@ script:
 - bash -c "sleep 2100; pkill ninja" &
 - "./tools/build.py -j2"
 - "./tools/test.py $DENO_BUILD_PATH"
+after_success:
+# TODO: Include hyperfine in //third_party
+- cargo install hyperfine
+- ./tools/benchmark.py $DENO_BUILD_PATH
+# export website assets for deployment
+- cp website/*.* gh-pages/
 before_deploy: |
   # gzip and name release to denote platform
   gzip -c $DENO_BUILD_PATH/deno > $DENO_BUILD_PATH/deno_${TRAVIS_OS_NAME}_x64.gz
 deploy:
-  provider: releases
+- provider: pages
+  local-dir: gh-pages
+  skip-cleanup: true
+  github-token: $GITHUB_TOKEN
+  keep-history: true
+  on:
+    branch: master
+- provider: releases
   api_key:
     secure: RIwv515oDcPAlEvt7uG8FeSFi6Tz6ODJUOXcFj6FYUPszxJ7Cg1kBLKln+fNW5OeOc52VsaZb/vPZ85skyEM6zk2ijL9FcSnnfNEm548w77iH6G0sk09NgBTy6KRXES6NZHD9jN1YTWYkT2G1NQi7mLqxR8a8pnWTbeK5HhtSWGsZPtXqf5iQbvnWsmKA0/w+FIgKupU0xe/qsYjh0eMLYpZDUWoKO0VxBKJ/ix5Uz91aJTjMIcHHij+ALg4pk+FkDotdyx39XB9b25KDxGuaI7NxWjSPzDxs/ZBHP6QYDLO0ti93ftvLAxRoBKPFoZrXqAu3KG9anr9WvxE40DO9OdV0VX2ZUatMUQm3DpSheN8ml2sErFqjIInqlpkdOVDYORz7FikPxkb9DKt+iuyFfxPRa4YWJv2tg8+Hy/nRCQw69OoKqrSNJ8KJDB3OjYbRBtdHz79RLJhTsGZla6RiyXfM7crR7CbFjbwdbW3Pt60t24fhvXQ0SwR0QTgzS/ieYEQHq/9GtSQA/Tn4kdIkyN6BdOMrQd/aUtgKmNdqbSlfmWGNyNZIxHdB+3RrTNT1tagkRI4UHEUfEujpIdYKwLjv0Xmi/VtTM+zOSkzHsIWGPfHBmIGnXfAItUHqivQYJ15E+dzg3T1CEbBxkDQtvwien9Fa8/pBsMkyovl8ps=
   file: "$DENO_BUILD_PATH/deno_${TRAVIS_OS_NAME}_x64.gz"

tools/benchmark.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# Performs benchmark and append data to //website/data.json.
+# If //website/data.json doesn't exist, this script tries to import it from gh-pages branch.
+# To view the results locally run ./tools/http_server.py and visit
+# https://localhost:4545/website
+
+import os
+import sys
+import json
+import time
+import shutil
+from util import run, run_output, root_path, build_path
+
+# The list of the tuples of the benchmark name and arguments
+benchmarks = [("hello", ["tests/002_hello.ts", "--reload"]),
+              ("relative_import", ["tests/003_relative_import.ts",
+                                   "--reload"])]
+
+gh_pages_data_file = "gh-pages/data.json"
+data_file = "website/data.json"
+
+
+def read_json(filename):
+    with open(filename) as json_file:
+        return json.load(json_file)
+
+
+def write_json(filename, data):
+    with open(filename, 'w') as outfile:
+        json.dump(data, outfile)
+
+
+def import_data_from_gh_pages():
+    if os.path.exists(data_file):
+        return
+    try:
+        run([
+            "git", "clone", "--depth", "1", "-b", "gh-pages",
+            "https://github.com/denoland/deno.git", "gh-pages"
+        ])
+        shutil.copy(gh_pages_data_file, data_file)
+    except:
+        write_json(data_file, [])  # writes empty json data
+
+
+def main(argv):
+    if len(argv) == 2:
+        build_dir = sys.argv[1]
+    elif len(argv) == 1:
+        build_dir = build_path()
+    else:
+        print "Usage: tools/benchmark.py [build_dir]"
+        sys.exit(1)
+
+    deno_path = os.path.join(build_dir, "deno")
+    benchmark_file = os.path.join(build_dir, "benchmark.json")
+
+    os.chdir(root_path)
+    import_data_from_gh_pages()
+    # TODO: Use hyperfine in //third_party
+    run(["hyperfine", "--export-json", benchmark_file, "--warmup", "3"] +
+        [deno_path + " " + " ".join(args) for [_, args] in benchmarks])
+    all_data = read_json(data_file)
+    benchmark_data = read_json(benchmark_file)
+    sha1 = run_output(["git", "rev-parse", "HEAD"]).strip()
+    new_data = {
+        "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "sha1": sha1,
+        "benchmark": {}
+    }
+    for [[name, _], data] in zip(benchmarks, benchmark_data["results"]):
+        new_data["benchmark"][name] = {
+            "mean": data["mean"],
+            "stddev": data["stddev"],
+            "user": data["user"],
+            "system": data["system"],
+            "min": data["min"],
+            "max": data["max"]
+        }
+    all_data.append(new_data)
+    write_json(data_file, all_data)
+
+
+if __name__ == '__main__':
+    main(sys.argv)

tools/format.py

@@ -29,7 +29,7 @@
 run(["yapf", "-i"] + glob("tools/*.py") + find_exts("build_extra", ".py"))
 
 run(["node", prettier, "--write"] + find_exts("js/", ".js", ".ts") +
-    find_exts("tests/", ".js", ".ts") +
+    find_exts("tests/", ".js", ".ts") + find_exts("website/", ".js", ".ts") +
     ["rollup.config.js", "tsconfig.json", "tslint.json"])
 
 # Requires rustfmt 0.8.2 (flags were different in previous versions)

website/app.js

@@ -0,0 +1,26 @@
+const benchmarkNames = ["hello", "relative_import"];
+
+(async () => {
+  const data = await (await fetch("./data.json")).json();
+
+  const benchmarkColumns = benchmarkNames.map(name => [
+    name,
+    ...data.map(d => {
+      const benchmark = d.benchmark[name];
+      return benchmark ? benchmark.mean : 0;
+    })
+  ]);
+
+  const sha1List = data.map(d => d.sha1);
+
+  c3.generate({
+    bindto: "#benchmark-chart",
+    data: { columns: benchmarkColumns },
+    axis: {
+      x: {
+        type: "category",
+        categories: sha1List
+      }
+    }
+  });
+})();

website/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>deno benchmark</title>
+  <link rel="stylesheet" href="https://unpkg.com/[email protected]/c3.min.css">
+</head>
+<body>
+  <div id="benchmark-chart"></div>
+  <script src="https://unpkg.com/[email protected]/dist/d3.min.js"></script>
+  <script src="https://unpkg.com/[email protected]/c3.min.js"></script>
+  <script src="./app.js"></script>
+</body>
+</html>
+