refactor(cli): cleanup compiler snapshot and tsc/module_graph (denola…

…nd#8220)

cli/Cargo.toml

@@ -23,6 +23,8 @@ path = "./bench/main.rs"
 deno_core = { path = "../core", version = "0.66.0" }
 deno_web = { path = "../op_crates/web", version = "0.17.0" }
 deno_fetch = { path = "../op_crates/fetch", version = "0.9.0" }
+regex = "1.3.9"
+serde = { version = "1.0.116", features = ["derive"] }
 
 [target.'cfg(windows)'.build-dependencies]
 winres = "0.1.11"

cli/build.rs

@@ -1,9 +1,13 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-mod op_fetch_asset;
-
+use deno_core::error::custom_error;
+use deno_core::json_op_sync;
+use deno_core::serde_json;
+use deno_core::serde_json::json;
 use deno_core::JsRuntime;
 use deno_core::RuntimeOptions;
+use regex::Regex;
+use serde::Deserialize;
 use std::collections::HashMap;
 use std::env;
 use std::path::Path;
@@ -46,46 +50,149 @@ fn create_runtime_snapshot(snapshot_path: &Path, files: Vec<PathBuf>) {
  create_snapshot(js_runtime, snapshot_path, files);
 }
 
+#[derive(Debug, Deserialize)]
+struct LoadArgs {
+ /// The fully qualified specifier that should be loaded.
+ specifier: String,
+}
+
 fn create_compiler_snapshot(
  snapshot_path: &Path,
  files: Vec<PathBuf>,
  cwd: &Path,
 ) {
- let mut custom_libs: HashMap<String, PathBuf> = HashMap::new();
- custom_libs
- .insert("lib.deno.web.d.ts".to_string(), deno_web::get_declaration());
- custom_libs.insert(
- "lib.deno.fetch.d.ts".to_string(),
- deno_fetch::get_declaration(),
- );
- custom_libs.insert(
- "lib.deno.window.d.ts".to_string(),
- cwd.join("dts/lib.deno.window.d.ts"),
- );
- custom_libs.insert(
- "lib.deno.worker.d.ts".to_string(),
- cwd.join("dts/lib.deno.worker.d.ts"),
- );
- custom_libs.insert(
- "lib.deno.shared_globals.d.ts".to_string(),
- cwd.join("dts/lib.deno.shared_globals.d.ts"),
- );
- custom_libs.insert(
- "lib.deno.ns.d.ts".to_string(),
- cwd.join("dts/lib.deno.ns.d.ts"),
- );
- custom_libs.insert(
- "lib.deno.unstable.d.ts".to_string(),
- cwd.join("dts/lib.deno.unstable.d.ts"),
- );
+ // libs that are being provided by op crates.
+ let mut op_crate_libs = HashMap::new();
+ op_crate_libs.insert("deno.web", deno_web::get_declaration());
+ op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration());
+
+ // ensure we invalidate the build properly.
+ for (_, path) in op_crate_libs.iter() {
+ println!("cargo:rerun-if-changed={}", path.display());
+ }
+
+ // libs that should be loaded into the isolate before snapshotting.
+ let libs = vec![
+ // Deno custom type libraries
+ "deno.window",
+ "deno.worker",
+ "deno.shared_globals",
+ "deno.ns",
+ "deno.unstable",
+ // Deno built-in type libraries
+ "es5",
+ "es2015.collection",
+ "es2015.core",
+ "es2015",
+ "es2015.generator",
+ "es2015.iterable",
+ "es2015.promise",
+ "es2015.proxy",
+ "es2015.reflect",
+ "es2015.symbol",
+ "es2015.symbol.wellknown",
+ "es2016.array.include",
+ "es2016",
+ "es2017",
+ "es2017.intl",
+ "es2017.object",
+ "es2017.sharedmemory",
+ "es2017.string",
+ "es2017.typedarrays",
+ "es2018.asyncgenerator",
+ "es2018.asynciterable",
+ "es2018",
+ "es2018.intl",
+ "es2018.promise",
+ "es2018.regexp",
+ "es2019.array",
+ "es2019",
+ "es2019.object",
+ "es2019.string",
+ "es2019.symbol",
+ "es2020.bigint",
+ "es2020",
+ "es2020.intl",
+ "es2020.promise",
+ "es2020.string",
+ "es2020.symbol.wellknown",
+ "esnext",
+ "esnext.intl",
+ "esnext.promise",
+ "esnext.string",
+ "esnext.weakref",
+ ];
+
+ // create a copy of the vector that includes any op crate libs to be passed
+ // to the JavaScript compiler to build into the snapshot
+ let mut build_libs = libs.clone();
+ for (op_lib, _) in op_crate_libs.iter() {
+ build_libs.push(op_lib.to_owned());
+ }
+
+ let re_asset = Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex");
+ let path_dts = cwd.join("dts");
+ let build_specifier = "asset:https:///bootstrap.ts";
 
  let mut js_runtime = JsRuntime::new(RuntimeOptions {
  will_snapshot: true,
  ..Default::default()
  });
  js_runtime.register_op(
- "op_fetch_asset",
- op_fetch_asset::op_fetch_asset(custom_libs),
+ "op_build_info",
+ json_op_sync(move |_state, _args, _bufs| {
+ Ok(json!({
+ "buildSpecifier": build_specifier,
+ "libs": build_libs,
+ }))
+ }),
+ );
+ // using the same op that is used in `tsc.rs` for loading modules and reading
+ // files, but a slightly different implementation at build time.
+ js_runtime.register_op(
+ "op_load",
+ json_op_sync(move |_state, args, _bufs| {
+ let v: LoadArgs = serde_json::from_value(args)?;
+ // we need a basic file to send to tsc to warm it up.
+ if v.specifier == build_specifier {
+ Ok(json!({
+ "data": r#"console.log("hello deno!");"#,
+ "hash": "1",
+ // this corresponds to `ts.ScriptKind.TypeScript`
+ "scriptKind": 3
+ }))
+ // specifiers come across as `asset:https:///lib.{lib_name}.d.ts` and we need to
+ // parse out just the name so we can lookup the asset.
+ } else if let Some(caps) = re_asset.captures(&v.specifier) {
+ if let Some(lib) = caps.get(1).map(|m| m.as_str()) {
+ // if it comes from an op crate, we were supplied with the path to the
+ // file.
+ let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) {
+ op_crate_lib.clone()
+ // otherwise we are will generate the path ourself
+ } else {
+ path_dts.join(format!("lib.{}.d.ts", lib))
+ };
+ let data = std::fs::read_to_string(path)?;
+ Ok(json!({
+ "data": data,
+ "hash": "1",
+ // this corresponds to `ts.ScriptKind.TypeScript`
+ "scriptKind": 3
+ }))
+ } else {
+ Err(custom_error(
+ "InvalidSpecifier",
+ format!("An invalid specifier was requested: {}", v.specifier),
+ ))
+ }
+ } else {
+ Err(custom_error(
+ "InvalidSpecifier",
+ format!("An invalid specifier was requested: {}", v.specifier),
+ ))
+ }
+ }),
  );
  create_snapshot(js_runtime, snapshot_path, files);
 }

cli/main.rs

@@ -35,9 +35,8 @@ mod lint;
 mod lockfile;
 mod media_type;
 mod metrics;
-mod module_graph2;
+mod module_graph;
 mod module_loader;
-mod op_fetch_asset;
 mod ops;
 mod permissions;
 mod program_state;
@@ -49,7 +48,7 @@ mod specifier_handler;
 mod test_runner;
 mod text_encoding;
 mod tokio_util;
-mod tsc2;
+mod tsc;
 mod tsc_config;
 mod upgrade;
 mod version;
@@ -177,7 +176,7 @@ async fn info_command(
  // so we allow access to all of them.
  Permissions::allow_all(),
  )?));
- let mut builder = module_graph2::GraphBuilder2::new(
+ let mut builder = module_graph::GraphBuilder::new(
  handler,
  program_state.maybe_import_map.clone(),
  program_state.lockfile.clone(),
@@ -241,9 +240,9 @@ async fn cache_command(
  files: Vec<String>,
 ) -> Result<(), AnyError> {
  let lib = if flags.unstable {
- module_graph2::TypeLib::UnstableDenoWindow
+ module_graph::TypeLib::UnstableDenoWindow
  } else {
- module_graph2::TypeLib::DenoWindow
+ module_graph::TypeLib::DenoWindow
  };
  let program_state = ProgramState::new(flags)?;
 
@@ -329,7 +328,7 @@ async fn bundle_command(
  // therefore we will allow the graph to access any module.
  Permissions::allow_all(),
  )?));
- let mut builder = module_graph2::GraphBuilder2::new(
+ let mut builder = module_graph::GraphBuilder::new(
  handler,
  program_state.maybe_import_map.clone(),
  program_state.lockfile.clone(),
@@ -341,12 +340,12 @@ async fn bundle_command(
  if !flags.no_check {
  // TODO(@kitsonk) support bundling for workers
  let lib = if flags.unstable {
- module_graph2::TypeLib::UnstableDenoWindow
+ module_graph::TypeLib::UnstableDenoWindow
  } else {
- module_graph2::TypeLib::DenoWindow
+ module_graph::TypeLib::DenoWindow
  };
  let graph = graph.clone();
- let result_info = graph.check(module_graph2::CheckOptions {
+ let result_info = graph.check(module_graph::CheckOptions {
  debug,
  emit: false,
  lib,
@@ -364,7 +363,7 @@ async fn bundle_command(
  }
 
  let (output, stats, maybe_ignored_options) =
- graph.bundle(module_graph2::BundleOptions {
+ graph.bundle(module_graph::BundleOptions {
  debug,
  maybe_config_path: flags.config_path,
  })?;
@@ -563,7 +562,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), AnyError> {
  &program_state,
  Permissions::allow_all(),
  )?));
- let mut builder = module_graph2::GraphBuilder2::new(
+ let mut builder = module_graph::GraphBuilder::new(
  handler,
  program_state.maybe_import_map.clone(),
  program_state.lockfile.clone(),