Compile cache and relative imports working.

.travis.yml

@@ -1,8 +1,12 @@
 language: go
 go:
  - 1.9.2
-cache: ccache
+cache:
+ directories:
+ - $HOME/.ccache
+ - `go env GOPATH`/src/github.com/ry/v8worker2/out
 install:
+ - go get github.com/jteeuwen/go-bindata
  - go get -d github.com/ry/v8worker2
  - (cd `go env GOPATH`/src/github.com/ry/v8worker2 && ./tools/build.py)
  - make

Makefile

@@ -1,3 +1,12 @@
+TS_FILES = \
+ amd.ts \
+ main.ts \
+ msg.pb.js \
+ compiler.ts \
+ msg.pb.d.ts \
+ os.ts \
+ util.ts
+
 deno: assets.go msg.pb.go main.go
  go build -o deno
 
@@ -13,7 +22,7 @@ msg.pb.js: msg.proto node_modules
 msg.pb.d.ts: msg.pb.js node_modules
  ./node_modules/.bin/pbts -o msg.pb.d.ts msg.pb.js
 
-dist/main.js: main.ts compiler.ts os.ts util.ts msg.pb.js msg.pb.d.ts node_modules
+dist/main.js: $(TS_FILES) node_modules
  ./node_modules/.bin/tsc --noEmit # Only for type checking.
  ./node_modules/.bin/parcel build --out-dir=dist/ --no-minify main.ts
 

amd.ts

@@ -0,0 +1,80 @@
+import * as path from "path";
+import { assert, log } from "./util";
+
+namespace ModuleExportsCache {
+ const cache = new Map<string, object>();
+ export function set(fileName: string, moduleExports: object) {
+ fileName = normalizeModuleName(fileName);
+ assert(
+ fileName.startsWith("/"),
+ `Normalized modules should start with /\n${fileName}`
+ );
+ log("ModuleExportsCache set", fileName);
+ cache.set(fileName, moduleExports);
+ }
+ export function get(fileName: string): object {
+ fileName = normalizeModuleName(fileName);
+ log("ModuleExportsCache get", fileName);
+ let moduleExports = cache.get(fileName);
+ if (moduleExports == null) {
+ moduleExports = {};
+ set(fileName, moduleExports);
+ }
+ return moduleExports;
+ }
+}
+
+function normalizeModuleName(fileName: string): string {
+ // Remove the extension.
+ return fileName.replace(/\.\w+$/, "");
+}
+
+function normalizeRelativeModuleName(contextFn: string, depFn: string): string {
+ if (depFn.startsWith("/")) {
+ return depFn;
+ } else {
+ return path.resolve(path.dirname(contextFn), depFn);
+ }
+}
+
+const executeQueue: Array<() => void> = [];
+
+export function executeQueueDrain(): void {
+ let fn;
+ while ((fn = executeQueue.shift())) {
+ fn();
+ }
+}
+
+// tslint:disable-next-line:no-any
+type AmdFactory = (...args: any[]) => undefined | object;
+type AmdDefine = (deps: string[], factory: AmdFactory) => void;
+
+export function makeDefine(fileName: string): AmdDefine {
+ const localDefine = (deps: string[], factory: AmdFactory): void => {
+ const localRequire = (x: string) => {
+ log("localRequire", x);
+ };
+ const localExports = ModuleExportsCache.get(fileName);
+ log("localDefine", fileName, deps, localExports);
+ const args = deps.map(dep => {
+ if (dep === "require") {
+ return localRequire;
+ } else if (dep === "exports") {
+ return localExports;
+ } else {
+ dep = normalizeRelativeModuleName(fileName, dep);
+ return ModuleExportsCache.get(dep);
+ }
+ });
+ executeQueue.push(() => {
+ log("execute", fileName);
+ const r = factory(...args);
+ if (r != null) {
+ ModuleExportsCache.set(fileName, r);
+ throw Error("x");
+ }
+ });
+ };
+ return localDefine;
+}

compiler.ts

@@ -1,19 +1,29 @@
 import * as ts from "typescript";
-import { log, assert, globalEval } from "./util";
-import { exit, readFileSync } from "./os";
+import { log, assert, globalEval, _global } from "./util";
+import * as os from "./os";
 import * as path from "path";
+import * as amd from "./amd";
+
+/*
+export function makeCacheDir(): string {
+ let cacheDir = path.join(env.HOME, ".deno/cache")
+ os.mkdirp(cacheDir);
+ return cacheDir
+}
+*/
 
 export function compile(cwd: string, inputFn: string): void {
  const options: ts.CompilerOptions = {
  allowJs: true,
- outDir: "_denoCache_/",
+ module: ts.ModuleKind.AMD,
+ outDir: "/" // Will be placed in ~/.deno/compile
  };
- const host = new CompilerHost(cwd);
+ const host = new CompilerHost();
 
  const inputExt = path.extname(inputFn);
  if (!EXTENSIONS.includes(inputExt)) {
  console.error(`Bad file name extension for input "${inputFn}"`);
- exit(1);
+ os.exit(1);
  }
 
  const program = ts.createProgram([inputFn], options, host);
@@ -27,12 +37,14 @@ export function compile(cwd: string, inputFn: string): void {
  for (const msg of errorMessages) {
  console.error(msg);
  }
- exit(2);
+ os.exit(2);
  }
 
  const emitResult = program.emit();
  assert(!emitResult.emitSkipped);
  log("emitResult", emitResult);
+
+ amd.executeQueueDrain();
 }
 
 /**
@@ -80,7 +92,7 @@ function getDiagnostics(program: ts.Program): ReadonlyArray<ts.Diagnostic> {
 const EXTENSIONS = [".ts", ".js"];
 
 export class CompilerHost {
- constructor(public cwd: string) {}
+ constructor() {}
 
  getSourceFile(
  fileName: string,
@@ -90,16 +102,14 @@ export class CompilerHost {
  ): ts.SourceFile | undefined {
  let sourceText: string;
  if (fileName === "lib.d.ts") {
- // TODO this should be compiled into the bindata.
- sourceText = readFileSync("node_modules/typescript/lib/lib.d.ts");
+ // TODO This should be compiled into the bindata.
+ sourceText = os.readFileSync("node_modules/typescript/lib/lib.d.ts");
  } else {
- sourceText = readFileSync(fileName);
+ sourceText = os.readFileSync(fileName);
  }
+ // fileName = fileName.replace(/\.\w+$/, ""); // Remove extension.
  if (sourceText) {
- log("getSourceFile", {
- fileName
- //sourceText,
- });
+ log("getSourceFile", { fileName });
  return ts.createSourceFile(fileName, sourceText, languageVersion);
  } else {
  log("getSourceFile NOT FOUND", { fileName });
@@ -134,13 +144,18 @@ export class CompilerHost {
  onError: ((message: string) => void) | undefined,
  sourceFiles: ReadonlyArray<ts.SourceFile>
  ): void {
- log("writeFile", { fileName, data });
+ //log("writeFile", { fileName, data });
+
+ os.compileOutput(data, fileName);
+
+ _global["define"] = amd.makeDefine(fileName);
  globalEval(data);
+ _global["define"] = null;
  }
 
  getCurrentDirectory(): string {
- log("getCurrentDirectory", this.cwd);
- return this.cwd;
+ log("getCurrentDirectory", ".");
+ return ".";
  }
 
  getDirectories(path: string): string[] {
@@ -165,7 +180,7 @@ export class CompilerHost {
  containingFile: string,
  reusedNames?: string[]
  ): Array<ts.ResolvedModule | undefined> {
- log("resolveModuleNames", { moduleNames, reusedNames });
+ //log("resolveModuleNames", { moduleNames, reusedNames });
  return moduleNames.map((name: string) => {
  if (
  name.startsWith("/") ||
@@ -177,7 +192,7 @@ export class CompilerHost {
  // Relative import.
  const containingDir = path.dirname(containingFile);
  const resolvedFileName = path.join(containingDir, name);
- log("relative import", { containingFile, name, resolvedFileName });
+ //log("relative import", { containingFile, name, resolvedFileName });
  const isExternalLibraryImport = false;
  return { resolvedFileName, isExternalLibraryImport };
  }

main.go

@@ -5,8 +5,28 @@ import (
  "github.com/ry/v8worker2"
  "io/ioutil"
  "os"
+ "path"
+ "path/filepath"
+ "runtime"
+ "strings"
 )
 
+func HandleCompileOutput(source string, filename string) []byte {
+ // println("compile output from golang", filename)
+ // Remove any ".." elements. This prevents hacking by trying to move up.
+ filename, err := filepath.Rel("/", filename)
+ check(err)
+ if strings.Contains(filename, "..") {
+ panic("Assertion error.")
+ }
+ filename = path.Join(CompileDir, filename)
+ err = os.MkdirAll(path.Dir(filename), 0700)
+ check(err)
+ err = ioutil.WriteFile(filename, []byte(source), 0600)
+ check(err)
+ return nil
+}
+
 func ReadFileSync(filename string) []byte {
  buf, err := ioutil.ReadFile(filename)
  msg := &Msg{Kind: Msg_DATA_RESPONSE}
@@ -16,57 +36,85 @@ func ReadFileSync(filename string) []byte {
  msg.Data = buf
  }
  out, err := proto.Marshal(msg)
- if err != nil {
- panic(err)
- }
+ check(err)
  return out
 }
 
+func UserHomeDir() string {
+ if runtime.GOOS == "windows" {
+ home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+ if home == "" {
+ home = os.Getenv("USERPROFILE")
+ }
+ return home
+ }
+ return os.Getenv("HOME")
+}
+
+func loadAsset(w *v8worker2.Worker, path string) {
+ data, err := Asset(path)
+ check(err)
+ err = w.Load(path, string(data))
+ check(err)
+}
+
+var DenoDir string
+var CompileDir string
+var SrcDir string
+
+func createDirs() {
+ DenoDir = path.Join(UserHomeDir(), ".deno")
+ CompileDir = path.Join(DenoDir, "compile")
+ err := os.MkdirAll(CompileDir, 0700)
+ check(err)
+ SrcDir = path.Join(DenoDir, "src")
+ err = os.MkdirAll(SrcDir, 0700)
+ check(err)
+}
+
+func check(e error) {
+ if e != nil {
+ panic(e)
+ }
+}
+
 func recv(buf []byte) []byte {
  msg := &Msg{}
  err := proto.Unmarshal(buf, msg)
- if err != nil {
- panic(err)
- }
+ check(err)
  switch msg.Kind {
  case Msg_READ_FILE_SYNC:
  return ReadFileSync(msg.Path)
  case Msg_EXIT:
  os.Exit(int(msg.Code))
+ case Msg_COMPILE_OUTPUT:
+ payload := msg.GetCompileOutput()
+ return HandleCompileOutput(payload.Source, payload.Filename)
  default:
  panic("Unexpected message")
  }
 
  return nil
 }
 
-func loadAsset(w *v8worker2.Worker, path string) {
- data, err := Asset(path)
- if err != nil {
- panic("asset not found")
- }
- err = w.Load(path, string(data))
- if err != nil {
- panic(err)
- }
-}
-
 func main() {
  args := v8worker2.SetFlags(os.Args)
+ createDirs()
  worker := v8worker2.New(recv)
  loadAsset(worker, "dist/main.js")
  cwd, err := os.Getwd()
- if err != nil {
- panic(err)
- }
+ check(err)
+
  out, err := proto.Marshal(&Msg{
  Kind: Msg_START,
- Cwd: cwd,
- Argv: args,
+ Payload: &Msg_Start{
+ Start: &StartMsg{
+ Cwd: cwd,
+ Argv: args,
+ },
+ },
  })
- if err != nil {
- panic(err)
- }
+ check(err)
  err = worker.SendBytes(out)
  if err != nil {
  os.Stderr.WriteString(err.Error())

main.ts

@@ -12,7 +12,7 @@ V8Worker2.recv((ab: ArrayBuffer) => {
  const msg = pb.Msg.decode(new Uint8Array(ab));
  switch (msg.kind) {
  case pb.Msg.MsgKind.START:
- start(msg.cwd, msg.argv);
+ start(msg.start.cwd, msg.start.argv);
  break;
  default:
  console.log("Unknown message", msg);