refactor(cli): cleanup compiler_snapshot and tsc/module_graph

cli/Cargo.toml

@@ -23,6 +23,8 @@ path = "./bench/main.rs"
 deno_core = { path = "../core", version = "0.66.0" }
 deno_web = { path = "../op_crates/web", version = "0.17.0" }
 deno_fetch = { path = "../op_crates/fetch", version = "0.9.0" }
+regex = "1.3.9"
+serde = { version = "1.0.116", features = ["derive"] }
 
 [target.'cfg(windows)'.build-dependencies]
 winres = "0.1.11"

cli/build.rs

@@ -1,9 +1,13 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-mod op_fetch_asset;
-
+use deno_core::error::custom_error;
+use deno_core::json_op_sync;
+use deno_core::serde_json;
+use deno_core::serde_json::json;
 use deno_core::JsRuntime;
 use deno_core::RuntimeOptions;
+use regex::Regex;
+use serde::Deserialize;
 use std::collections::HashMap;
 use std::env;
 use std::path::Path;
@@ -46,46 +50,139 @@ fn create_runtime_snapshot(snapshot_path: &Path, files: Vec<PathBuf>) {
   create_snapshot(js_runtime, snapshot_path, files);
 }
 
+#[derive(Debug, Deserialize)]
+struct LoadArgs {
+  /// The fully qualified specifier that should be loaded.
+  specifier: String,
+}
+
 fn create_compiler_snapshot(
   snapshot_path: &Path,
   files: Vec<PathBuf>,
   cwd: &Path,
 ) {
-  let mut custom_libs: HashMap<String, PathBuf> = HashMap::new();
-  custom_libs
-    .insert("lib.deno.web.d.ts".to_string(), deno_web::get_declaration());
-  custom_libs.insert(
-    "lib.deno.fetch.d.ts".to_string(),
-    deno_fetch::get_declaration(),
-  );
-  custom_libs.insert(
-    "lib.deno.window.d.ts".to_string(),
-    cwd.join("dts/lib.deno.window.d.ts"),
-  );
-  custom_libs.insert(
-    "lib.deno.worker.d.ts".to_string(),
-    cwd.join("dts/lib.deno.worker.d.ts"),
-  );
-  custom_libs.insert(
-    "lib.deno.shared_globals.d.ts".to_string(),
-    cwd.join("dts/lib.deno.shared_globals.d.ts"),
-  );
-  custom_libs.insert(
-    "lib.deno.ns.d.ts".to_string(),
-    cwd.join("dts/lib.deno.ns.d.ts"),
-  );
-  custom_libs.insert(
-    "lib.deno.unstable.d.ts".to_string(),
-    cwd.join("dts/lib.deno.unstable.d.ts"),
-  );
+  // libs that are being provided by op crates.
+  let mut op_crate_libs = HashMap::new();
+  op_crate_libs.insert("deno.web", deno_web::get_declaration());
+  op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration());
+
+  // ensure we invalidate the build properly.
+  for (_, path) in op_crate_libs.iter() {
+    println!("cargo:rerun-if-changed={}", path.display());
+  }
+
+  // libs that should be loaded into the isolate before snapshotting.
+  let libs = vec![
+    // Deno custom type libraries
+    "deno.window",
+    "deno.worker",
+    "deno.shared_globals",
+    "deno.ns",
+    "deno.unstable",
+    // Deno built-in type libraries
+    "es5",
+    "es2015.collection",
+    "es2015.core",
+    "es2015",
+    "es2015.generator",
+    "es2015.iterable",
+    "es2015.promise",
+    "es2015.proxy",
+    "es2015.reflect",
+    "es2015.symbol",
+    "es2015.symbol.wellknown",
+    "es2016.array.include",
+    "es2016",
+    "es2017",
+    "es2017.intl",
+    "es2017.object",
+    "es2017.sharedmemory",
+    "es2017.string",
+    "es2017.typedarrays",
+    "es2018.asyncgenerator",
+    "es2018.asynciterable",
+    "es2018",
+    "es2018.intl",
+    "es2018.promise",
+    "es2018.regexp",
+    "es2019.array",
+    "es2019",
+    "es2019.object",
+    "es2019.string",
+    "es2019.symbol",
+    "es2020.bigint",
+    "es2020",
+    "es2020.intl",
+    "es2020.promise",
+    "es2020.string",
+    "es2020.symbol.wellknown",
+    "esnext",
+    "esnext.intl",
+    "esnext.promise",
+    "esnext.string",
+    "esnext.weakref",
+  ];
+
+  // create a copy of the vector that includes any op crate libs to be passed
+  // to the JavaScript compiler to build into the snapshot
+  let mut build_libs = libs.clone();
+  for (op_lib, _) in op_crate_libs.iter() {
+    build_libs.push(&(*op_lib).clone());
+  }
+
+  let re_asset = Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex");
+  let path_dts = cwd.join("dts");
+  let build_specifier = "asset:https:///bootstrap.ts";
 
   let mut js_runtime = JsRuntime::new(RuntimeOptions {
     will_snapshot: true,
     ..Default::default()
   });
   js_runtime.register_op(
-    "op_fetch_asset",
-    op_fetch_asset::op_fetch_asset(custom_libs),
+    "op_build_info",
+    json_op_sync(move |_state, _args, _bufs| {
+      Ok(json!({
+        "buildSpecifier": build_specifier,
+        "libs": build_libs,
+      }))
+    }),
+  );
+  js_runtime.register_op(
+    "op_load",
+    json_op_sync(move |_state, args, _bufs| {
+      let v: LoadArgs = serde_json::from_value(args)?;
+      if v.specifier == build_specifier {
+        Ok(json!({
+          "data": r#"console.log("hello deno!");"#,
+          "hash": "1",
+          "scriptKind": 3
+        }))
+      } else if let Some(caps) = re_asset.captures(&v.specifier) {
+        if let Some(lib) = caps.get(1).map(|m| m.as_str()) {
+          let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) {
+            op_crate_lib.clone()
+          } else {
+            path_dts.join(format!("lib.{}.d.ts", lib))
+          };
+          let data = std::fs::read_to_string(path)?;
+          Ok(json!({
+            "data": data,
+            "hash": "1",
+            "scriptKind": 3
+          }))
+        } else {
+          Err(custom_error(
+            "InvalidSpecifier",
+            format!("An invalid specifier was requested: {}", v.specifier),
+          ))
+        }
+      } else {
+        Err(custom_error(
+          "InvalidSpecifier",
+          format!("An invalid specifier was requested: {}", v.specifier),
+        ))
+      }
+    }),
   );
   create_snapshot(js_runtime, snapshot_path, files);
 }

cli/main.rs

@@ -35,9 +35,8 @@ mod lint;
 mod lockfile;
 mod media_type;
 mod metrics;
-mod module_graph2;
+mod module_graph;
 mod module_loader;
-mod op_fetch_asset;
 mod ops;
 mod permissions;
 mod program_state;
@@ -49,7 +48,7 @@ mod specifier_handler;
 mod test_runner;
 mod text_encoding;
 mod tokio_util;
-mod tsc2;
+mod tsc;
 mod tsc_config;
 mod upgrade;
 mod version;
@@ -177,7 +176,7 @@ async fn info_command(
       // so we allow access to all of them.
       Permissions::allow_all(),
     )?));
-    let mut builder = module_graph2::GraphBuilder2::new(
+    let mut builder = module_graph::GraphBuilder::new(
       handler,
       program_state.maybe_import_map.clone(),
       program_state.lockfile.clone(),
@@ -241,9 +240,9 @@ async fn cache_command(
   files: Vec<String>,
 ) -> Result<(), AnyError> {
   let lib = if flags.unstable {
-    module_graph2::TypeLib::UnstableDenoWindow
+    module_graph::TypeLib::UnstableDenoWindow
   } else {
-    module_graph2::TypeLib::DenoWindow
+    module_graph::TypeLib::DenoWindow
   };
   let program_state = ProgramState::new(flags)?;
 
@@ -329,7 +328,7 @@ async fn bundle_command(
     // therefore we will allow the graph to access any module.
     Permissions::allow_all(),
   )?));
-  let mut builder = module_graph2::GraphBuilder2::new(
+  let mut builder = module_graph::GraphBuilder::new(
     handler,
     program_state.maybe_import_map.clone(),
     program_state.lockfile.clone(),
@@ -341,12 +340,12 @@ async fn bundle_command(
   if !flags.no_check {
     // TODO(@kitsonk) support bundling for workers
     let lib = if flags.unstable {
-      module_graph2::TypeLib::UnstableDenoWindow
+      module_graph::TypeLib::UnstableDenoWindow
     } else {
-      module_graph2::TypeLib::DenoWindow
+      module_graph::TypeLib::DenoWindow
     };
     let graph = graph.clone();
-    let result_info = graph.check(module_graph2::CheckOptions {
+    let result_info = graph.check(module_graph::CheckOptions {
       debug,
       emit: false,
       lib,
@@ -364,7 +363,7 @@ async fn bundle_command(
   }
 
   let (output, stats, maybe_ignored_options) =
-    graph.bundle(module_graph2::BundleOptions {
+    graph.bundle(module_graph::BundleOptions {
       debug,
       maybe_config_path: flags.config_path,
     })?;
@@ -563,7 +562,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), AnyError> {
     &program_state,
     Permissions::allow_all(),
   )?));
-  let mut builder = module_graph2::GraphBuilder2::new(
+  let mut builder = module_graph::GraphBuilder::new(
     handler,
     program_state.maybe_import_map.clone(),
     program_state.lockfile.clone(),