Execute JS for the first time in Rust rewrite.

Implements code_fetch handler in Rust.

Add ability to embed string assets (for typescript declaration files)

Remove deno_cc and deno_cc_nosnapshot targets.

.travis.yml

@@ -47,7 +47,7 @@ script:
  - ./tools/lint.py
  - $BUILD_PATH/test_cc
  - $BUILD_PATH/handlers_test
- - $BUILD_PATH/deno_cc foo bar
- - $BUILD_PATH/deno_cc_nosnapshot foo bar
- - $BUILD_PATH/deno meow
- - $BUILD_PATH/deno_nosnapshot meow
+ - $BUILD_PATH/deno ./testdata/001_hello.js
+ - $BUILD_PATH/deno ./testdata/002_hello.ts
+ - $BUILD_PATH/deno_nosnapshot ./testdata/001_hello.js
+ - $BUILD_PATH/deno_nosnapshot ./testdata/002_hello.ts

BUILD.gn

@@ -8,8 +8,6 @@ group("all") {
  testonly = true
  deps = [
  ":deno",
- ":deno_cc",
- ":deno_cc_nosnapshot",
  ":deno_nosnapshot",
  ":handlers_test",
  ":test_cc",
@@ -61,35 +59,6 @@ rust_test("handlers_test") {
  ]
 }
 
-executable("deno_cc") {
- sources = [
- "src/main.cc",
- ]
- deps = [
- ":flatbufferjs",
- ":handlers",
- ":libdeno",
- ":msg_cpp",
- ]
- configs += [ ":deno_config" ]
-}
-
-# This target is for fast incremental development.
-# When modifying the javascript runtime, this target will not go through the
-# extra process of building a snapshot and instead load the bundle from disk.
-executable("deno_cc_nosnapshot") {
- sources = [
- "src/main.cc",
- ]
- deps = [
- ":flatbufferjs",
- ":handlers",
- ":libdeno_nosnapshot",
- ":msg_cpp",
- ]
- configs += [ ":deno_config" ]
-}
-
 executable("test_cc") {
  testonly = true
  sources = [
@@ -154,9 +123,12 @@ v8_source_set("deno_bindings") {
  sources = [
  "src/flatbuffer_builder.cc",
  "src/flatbuffer_builder.h",
+ "src/reply.cc",
+ "src/reply.h",
  ]
  deps = [
  ":deno_base",
+ ":handlers",
  ":msg_cpp",
  ]
  public_deps = [
@@ -184,6 +156,7 @@ flatbuffer("msg_cpp") {
 run_node("bundle") {
  out_dir = "$target_gen_dir/bundle/"
  sources = [
+ "js/assets.ts",
  "js/console.ts",
  "js/deno.d.ts",
  "js/dispatch.ts",

js/assets.ts

@@ -0,0 +1,51 @@
+// Copyright 2018 Ryan Dahl <[email protected]>
+// All rights reserved. MIT License.
+
+// This file is formatted as it is because we are using the fact that Parcel
+// statically evaluates fs.readFileSync.
+import { readFileSync } from "fs";
+
+// tslint:disable:max-line-length
+// prettier-ignore
+export const assetSourceCode: { [key: string]: string } = {
+ "deno.d.ts": readFileSync(__dirname + "/deno.d.ts", "utf8"),
+ "lib.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.d.ts", "utf8"),
+ //"lib.dom.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.dom.d.ts", "utf8"),
+ "lib.dom.iterable.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.dom.iterable.d.ts", "utf8"),
+ "lib.es2015.collection.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.collection.d.ts", "utf8"),
+ "lib.es2015.core.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.core.d.ts", "utf8"),
+ //"lib.es2015.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.d.ts", "utf8"),
+ "lib.es2015.generator.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.generator.d.ts", "utf8"),
+ "lib.es2015.iterable.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.iterable.d.ts", "utf8"),
+ "lib.es2015.promise.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.promise.d.ts", "utf8"),
+ "lib.es2015.proxy.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.proxy.d.ts", "utf8"),
+ "lib.es2015.reflect.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.reflect.d.ts", "utf8"),
+ "lib.es2015.symbol.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.symbol.d.ts", "utf8"),
+ "lib.es2015.symbol.wellknown.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts", "utf8"),
+ "lib.es2016.array.include.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2016.array.include.d.ts", "utf8"),
+ //"lib.es2016.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2016.d.ts", "utf8"),
+ //"lib.es2016.full.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2016.full.d.ts", "utf8"),
+ //"lib.es2017.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.d.ts", "utf8"),
+ //"lib.es2017.full.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.full.d.ts", "utf8"),
+ "lib.es2017.intl.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.intl.d.ts", "utf8"),
+ "lib.es2017.object.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.object.d.ts", "utf8"),
+ "lib.es2017.sharedmemory.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.sharedmemory.d.ts", "utf8"),
+ "lib.es2017.string.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.string.d.ts", "utf8"),
+ "lib.es2017.typedarrays.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.typedarrays.d.ts", "utf8"),
+ "lib.es2018.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2018.d.ts", "utf8"),
+ //"lib.es2018.full.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2018.full.d.ts", "utf8"),
+ "lib.es2018.promise.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2018.promise.d.ts", "utf8"),
+ "lib.es2018.regexp.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2018.regexp.d.ts", "utf8"),
+ //"lib.es5.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es5.d.ts", "utf8"),
+ //"lib.es6.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es6.d.ts", "utf8"),
+ "lib.esnext.array.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.esnext.array.d.ts", "utf8"),
+ "lib.esnext.asynciterable.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.esnext.asynciterable.d.ts", "utf8"),
+ "lib.esnext.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.esnext.d.ts", "utf8"),
+ //"lib.esnext.full.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.esnext.full.d.ts", "utf8"),
+ //"lib.scripthost.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.scripthost.d.ts", "utf8"),
+ //"lib.webworker.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.webworker.d.ts", "utf8"),
+ //"protocol.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/protocol.d.ts", "utf8"),
+ //"tsserverlibrary.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/tsserverlibrary.d.ts", "utf8"),
+ "typescript.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/typescript.d.ts", "utf8"),
+ //"typescriptServices.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/typescriptServices.d.ts", "utf8"),
+};

js/main.ts

@@ -1,12 +1,9 @@
 // tslint:disable-next-line:no-reference
 /// <reference path="deno.d.ts" />
-import * as ts from "typescript";
-
 import { flatbuffers } from "flatbuffers";
 import { deno as fbs } from "./msg_generated";
-import { assert } from "./util";
-
-// import * as runtime from "./runtime";
+import { assert, log } from "./util";
+import * as runtime from "./runtime";
 
 const globalEval = eval;
 const window = globalEval("this");
@@ -31,8 +28,6 @@ function startMsg(cmdId: number): Uint8Array {
 }
 
 window["denoMain"] = () => {
- deno.print(`ts.version: ${ts.version}`);
-
  // First we send an empty "Start" message to let the privlaged side know we
  // are ready. The response should be a "StartRes" message containing the CLI
  // argv and other info.
@@ -55,17 +50,15 @@ window["denoMain"] = () => {
  assert(base.msg(startResMsg) != null);
 
  const cwd = startResMsg.cwd();
- deno.print(`cwd: ${cwd}`);
+ log("cwd", cwd);
 
  const argv: string[] = [];
  for (let i = 0; i < startResMsg.argvLength(); i++) {
  argv.push(startResMsg.argv(i));
  }
- deno.print(`argv ${argv}`);
+ log("argv", argv);
 
- /* TODO(ry) Uncomment to test further message passing.
- const inputFn = argv[0];
+ const inputFn = argv[1];
  const mod = runtime.resolveModule(inputFn, `${cwd}/`);
  mod.compileAndRun();
- */
 };

js/os.ts

@@ -2,7 +2,8 @@
 // All rights reserved. MIT License.
 import { ModuleInfo } from "./types";
 import { deno as fbs } from "./msg_generated";
-import { assert, typedArrayToArrayBuffer } from "./util";
+import { assert } from "./util";
+import * as util from "./util";
 import { flatbuffers } from "flatbuffers";
 
 export function exit(exitCode = 0): void {
@@ -19,8 +20,7 @@ export function codeFetch(
  moduleSpecifier: string,
  containingFile: string
 ): ModuleInfo {
- console.log("Hello from codeFetch");
-
+ util.log("os.ts codeFetch", moduleSpecifier, containingFile);
  // Send CodeFetch message
  const builder = new flatbuffers.Builder();
  const moduleSpecifier_ = builder.createString(moduleSpecifier);
@@ -33,52 +33,46 @@ export function codeFetch(
  fbs.Base.addMsg(builder, msg);
  fbs.Base.addMsgType(builder, fbs.Any.CodeFetch);
  builder.finish(fbs.Base.endBase(builder));
- const payload = typedArrayToArrayBuffer(builder.asUint8Array());
- const resBuf = deno.send("x", payload);
-
- console.log("CodeFetch sent");
-
+ const resBuf = deno.send(builder.asUint8Array());
  // Process CodeFetchRes
  const bb = new flatbuffers.ByteBuffer(new Uint8Array(resBuf));
  const baseRes = fbs.Base.getRootAsBase(bb);
+ if (fbs.Any.NONE === baseRes.msgType()) {
+ throw Error(baseRes.error());
+ }
  assert(fbs.Any.CodeFetchRes === baseRes.msgType());
  const codeFetchRes = new fbs.CodeFetchRes();
  assert(baseRes.msg(codeFetchRes) != null);
- return {
+ const r = {
  moduleName: codeFetchRes.moduleName(),
  filename: codeFetchRes.filename(),
  sourceCode: codeFetchRes.sourceCode(),
  outputCode: codeFetchRes.outputCode()
  };
+ return r;
 }
 
 export function codeCache(
  filename: string,
  sourceCode: string,
  outputCode: string
 ): void {
+ util.log("os.ts codeCache", filename, sourceCode, outputCode);
  const builder = new flatbuffers.Builder();
-
  const filename_ = builder.createString(filename);
  const sourceCode_ = builder.createString(sourceCode);
  const outputCode_ = builder.createString(outputCode);
-
  fbs.CodeCache.startCodeCache(builder);
  fbs.CodeCache.addFilename(builder, filename_);
  fbs.CodeCache.addSourceCode(builder, sourceCode_);
  fbs.CodeCache.addOutputCode(builder, outputCode_);
  const msg = fbs.CodeCache.endCodeCache(builder);
-
  fbs.Base.startBase(builder);
  fbs.Base.addMsg(builder, msg);
+ fbs.Base.addMsgType(builder, fbs.Any.CodeCache);
  builder.finish(fbs.Base.endBase(builder));
-
- // Maybe need to do another step?
- // Base.finishBaseBuffer(builder, base);
-
- const payload = typedArrayToArrayBuffer(builder.asUint8Array());
- const resBuf = deno.send("x", payload);
- assert(resBuf === null);
+ const resBuf = deno.send(builder.asUint8Array());
+ assert(resBuf == null);
 }
 
 export function readFileSync(filename: string): Uint8Array {

js/runtime.ts

@@ -11,6 +11,7 @@
 import * as ts from "typescript";
 import * as util from "./util";
 import { log } from "./util";
+import { assetSourceCode } from "./assets";
 import * as os from "./os";
 //import * as sourceMaps from "./v8_source_maps";
 import { window, globalEval } from "./globals";
@@ -22,9 +23,14 @@ const EOL = "\n";
 export type AmdFactory = (...args: any[]) => undefined | object;
 export type AmdDefine = (deps: string[], factory: AmdFactory) => void;
 
-/*
 // Uncaught exceptions are sent to window.onerror by the privlaged binding.
-window.onerror = (message, source, lineno, colno, error) => {
+window.onerror = (
+ message: string,
+ source: string,
+ lineno: number,
+ colno: number,
+ error: Error
+) => {
  // TODO Currently there is a bug in v8_source_maps.ts that causes a segfault
  // if it is used within window.onerror. To workaround we uninstall the
  // Error.prepareStackTrace handler. Users will get unmapped stack traces on
@@ -33,7 +39,6 @@ window.onerror = (message, source, lineno, colno, error) => {
  console.log(error.message, error.stack);
  os.exit(1);
 };
-*/
 
 /*
 export function setup(mainJs: string, mainMap: string): void {
@@ -147,11 +152,31 @@ export function resolveModule(
 ): null | FileModule {
  util.log("resolveModule", { moduleSpecifier, containingFile });
  util.assert(moduleSpecifier != null && moduleSpecifier.length > 0);
- // We ask golang to sourceCodeFetch. It will load the sourceCode and if
- // there is any outputCode cached, it will return that as well.
- const fetchResponse = os.codeFetch(moduleSpecifier, containingFile);
- const { filename, sourceCode, outputCode } = fetchResponse;
- if (sourceCode.length === 0) {
+ let filename: string, sourceCode: string, outputCode: string;
+ if (
+ moduleSpecifier.startsWith("/$asset$/") ||
+ containingFile.startsWith("/$asset$/")
+ ) {
+ // Assets are compiled into the runtime javascript bundle.
+ const assetName = moduleSpecifier.split("/").pop();
+ sourceCode = assetSourceCode[assetName];
+ filename = "/$asset$/" + assetName;
+ } else {
+ // We query Rust with a CodeFetch message. It will load the sourceCode, and
+ // if there is any outputCode cached, will return that as well.
+ let fetchResponse;
+ try {
+ fetchResponse = os.codeFetch(moduleSpecifier, containingFile);
+ } catch (e) {
+ // TODO Only catch "no such file or directory" errors. Need error codes.
+ util.log("os.codeFetch error ignored", e.message);
+ return null;
+ }
+ filename = fetchResponse.filename;
+ sourceCode = fetchResponse.sourceCode;
+ outputCode = fetchResponse.outputCode;
+ }
+ if (sourceCode == null || sourceCode.length === 0) {
  return null;
  }
  util.log("resolveModule sourceCode length ", sourceCode.length);
@@ -293,7 +318,7 @@ class TypeScriptHost implements ts.LanguageServiceHost {
  }
 
  getDefaultLibFileName(options: ts.CompilerOptions): string {
- const fn = ts.getDefaultLibFileName(options);
+ const fn = "lib.d.ts"; // ts.getDefaultLibFileName(options);
  util.log("getDefaultLibFileName", fn);
  const m = resolveModule(fn, "/$asset$/");
  return m.fileName;

js/util.ts

@@ -2,7 +2,7 @@
 // All rights reserved. MIT License.
 
 //import { debug } from "./main";
-const debug = true;
+const debug = false;
 
 import { TypedArray } from "./types";
 

src/binding.cc

@@ -301,8 +301,20 @@ void deno_init() {
 
 const char* deno_v8_version() { return v8::V8::GetVersion(); }
 
+// TODO(ry) Remove these when we call deno_reply_start from Rust.
+static char** global_argv;
+static int global_argc;
+char** deno_argv() { return global_argv; }
+int deno_argc() { return global_argc; }
+
 void deno_set_flags(int* argc, char** argv) {
  v8::V8::SetFlagsFromCommandLine(argc, argv, true);
+ // TODO(ry) Remove these when we call deno_reply_start from Rust.
+ global_argc = *argc;
+ global_argv = reinterpret_cast<char**>(malloc(*argc * sizeof(char*)));
+ for (int i = 0; i < *argc; i++) {
+ global_argv[i] = strdup(argv[i]);
+ }
 }
 
 const char* deno_last_exception(Deno* d) { return d->last_exception.c_str(); }

src/handlers.h

@@ -2,10 +2,12 @@
 // All rights reserved. MIT License.
 #ifndef HANDLERS_H_
 #define HANDLERS_H_
-extern "C" {
+
 #include <stdint.h>
+#include "deno.h"
 
-void handle_code_fetch(uint32_t cmd_id, const char* module_specifier,
+extern "C" {
+void handle_code_fetch(Deno* d, uint32_t cmd_id, const char* module_specifier,
  const char* containing_file);
-}
+} // extern "C"
 #endif // HANDLERS_H_