Skip to content

Commit

Permalink
refactor(cli): cleanup compiler snapshot and tsc/module_graph (denola…
Browse files Browse the repository at this point in the history
  • Loading branch information
kitsonk authored and jannes committed Dec 1, 2020
1 parent 690e7d7 commit 6c743ed
Show file tree
Hide file tree
Showing 10 changed files with 368 additions and 644 deletions.
2 changes: 2 additions & 0 deletions cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ path = "./bench/main.rs"
deno_core = { path = "../core", version = "0.66.0" }
deno_web = { path = "../op_crates/web", version = "0.17.0" }
deno_fetch = { path = "../op_crates/fetch", version = "0.9.0" }
regex = "1.3.9"
serde = { version = "1.0.116", features = ["derive"] }

[target.'cfg(windows)'.build-dependencies]
winres = "0.1.11"
Expand Down
169 changes: 138 additions & 31 deletions cli/build.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.

mod op_fetch_asset;

use deno_core::error::custom_error;
use deno_core::json_op_sync;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::JsRuntime;
use deno_core::RuntimeOptions;
use regex::Regex;
use serde::Deserialize;
use std::collections::HashMap;
use std::env;
use std::path::Path;
Expand Down Expand Up @@ -46,46 +50,149 @@ fn create_runtime_snapshot(snapshot_path: &Path, files: Vec<PathBuf>) {
create_snapshot(js_runtime, snapshot_path, files);
}

#[derive(Debug, Deserialize)]
struct LoadArgs {
/// The fully qualified specifier that should be loaded.
specifier: String,
}

fn create_compiler_snapshot(
snapshot_path: &Path,
files: Vec<PathBuf>,
cwd: &Path,
) {
let mut custom_libs: HashMap<String, PathBuf> = HashMap::new();
custom_libs
.insert("lib.deno.web.d.ts".to_string(), deno_web::get_declaration());
custom_libs.insert(
"lib.deno.fetch.d.ts".to_string(),
deno_fetch::get_declaration(),
);
custom_libs.insert(
"lib.deno.window.d.ts".to_string(),
cwd.join("dts/lib.deno.window.d.ts"),
);
custom_libs.insert(
"lib.deno.worker.d.ts".to_string(),
cwd.join("dts/lib.deno.worker.d.ts"),
);
custom_libs.insert(
"lib.deno.shared_globals.d.ts".to_string(),
cwd.join("dts/lib.deno.shared_globals.d.ts"),
);
custom_libs.insert(
"lib.deno.ns.d.ts".to_string(),
cwd.join("dts/lib.deno.ns.d.ts"),
);
custom_libs.insert(
"lib.deno.unstable.d.ts".to_string(),
cwd.join("dts/lib.deno.unstable.d.ts"),
);
// libs that are being provided by op crates.
let mut op_crate_libs = HashMap::new();
op_crate_libs.insert("deno.web", deno_web::get_declaration());
op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration());

// ensure we invalidate the build properly.
for (_, path) in op_crate_libs.iter() {
println!("cargo:rerun-if-changed={}", path.display());
}

// libs that should be loaded into the isolate before snapshotting.
let libs = vec![
// Deno custom type libraries
"deno.window",
"deno.worker",
"deno.shared_globals",
"deno.ns",
"deno.unstable",
// Deno built-in type libraries
"es5",
"es2015.collection",
"es2015.core",
"es2015",
"es2015.generator",
"es2015.iterable",
"es2015.promise",
"es2015.proxy",
"es2015.reflect",
"es2015.symbol",
"es2015.symbol.wellknown",
"es2016.array.include",
"es2016",
"es2017",
"es2017.intl",
"es2017.object",
"es2017.sharedmemory",
"es2017.string",
"es2017.typedarrays",
"es2018.asyncgenerator",
"es2018.asynciterable",
"es2018",
"es2018.intl",
"es2018.promise",
"es2018.regexp",
"es2019.array",
"es2019",
"es2019.object",
"es2019.string",
"es2019.symbol",
"es2020.bigint",
"es2020",
"es2020.intl",
"es2020.promise",
"es2020.string",
"es2020.symbol.wellknown",
"esnext",
"esnext.intl",
"esnext.promise",
"esnext.string",
"esnext.weakref",
];

// create a copy of the vector that includes any op crate libs to be passed
// to the JavaScript compiler to build into the snapshot
let mut build_libs = libs.clone();
for (op_lib, _) in op_crate_libs.iter() {
build_libs.push(op_lib.to_owned());
}

let re_asset = Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex");
let path_dts = cwd.join("dts");
let build_specifier = "asset:https:///bootstrap.ts";

let mut js_runtime = JsRuntime::new(RuntimeOptions {
will_snapshot: true,
..Default::default()
});
js_runtime.register_op(
"op_fetch_asset",
op_fetch_asset::op_fetch_asset(custom_libs),
"op_build_info",
json_op_sync(move |_state, _args, _bufs| {
Ok(json!({
"buildSpecifier": build_specifier,
"libs": build_libs,
}))
}),
);
// using the same op that is used in `tsc.rs` for loading modules and reading
// files, but a slightly different implementation at build time.
js_runtime.register_op(
"op_load",
json_op_sync(move |_state, args, _bufs| {
let v: LoadArgs = serde_json::from_value(args)?;
// we need a basic file to send to tsc to warm it up.
if v.specifier == build_specifier {
Ok(json!({
"data": r#"console.log("hello deno!");"#,
"hash": "1",
// this corresponds to `ts.ScriptKind.TypeScript`
"scriptKind": 3
}))
// specifiers come across as `asset:https:///lib.{lib_name}.d.ts` and we need to
// parse out just the name so we can lookup the asset.
} else if let Some(caps) = re_asset.captures(&v.specifier) {
if let Some(lib) = caps.get(1).map(|m| m.as_str()) {
// if it comes from an op crate, we were supplied with the path to the
// file.
let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) {
op_crate_lib.clone()
// otherwise we are will generate the path ourself
} else {
path_dts.join(format!("lib.{}.d.ts", lib))
};
let data = std::fs::read_to_string(path)?;
Ok(json!({
"data": data,
"hash": "1",
// this corresponds to `ts.ScriptKind.TypeScript`
"scriptKind": 3
}))
} else {
Err(custom_error(
"InvalidSpecifier",
format!("An invalid specifier was requested: {}", v.specifier),
))
}
} else {
Err(custom_error(
"InvalidSpecifier",
format!("An invalid specifier was requested: {}", v.specifier),
))
}
}),
);
create_snapshot(js_runtime, snapshot_path, files);
}
Expand Down
23 changes: 11 additions & 12 deletions cli/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,8 @@ mod lint;
mod lockfile;
mod media_type;
mod metrics;
mod module_graph2;
mod module_graph;
mod module_loader;
mod op_fetch_asset;
mod ops;
mod permissions;
mod program_state;
Expand All @@ -49,7 +48,7 @@ mod specifier_handler;
mod test_runner;
mod text_encoding;
mod tokio_util;
mod tsc2;
mod tsc;
mod tsc_config;
mod upgrade;
mod version;
Expand Down Expand Up @@ -177,7 +176,7 @@ async fn info_command(
// so we allow access to all of them.
Permissions::allow_all(),
)?));
let mut builder = module_graph2::GraphBuilder2::new(
let mut builder = module_graph::GraphBuilder::new(
handler,
program_state.maybe_import_map.clone(),
program_state.lockfile.clone(),
Expand Down Expand Up @@ -241,9 +240,9 @@ async fn cache_command(
files: Vec<String>,
) -> Result<(), AnyError> {
let lib = if flags.unstable {
module_graph2::TypeLib::UnstableDenoWindow
module_graph::TypeLib::UnstableDenoWindow
} else {
module_graph2::TypeLib::DenoWindow
module_graph::TypeLib::DenoWindow
};
let program_state = ProgramState::new(flags)?;

Expand Down Expand Up @@ -329,7 +328,7 @@ async fn bundle_command(
// therefore we will allow the graph to access any module.
Permissions::allow_all(),
)?));
let mut builder = module_graph2::GraphBuilder2::new(
let mut builder = module_graph::GraphBuilder::new(
handler,
program_state.maybe_import_map.clone(),
program_state.lockfile.clone(),
Expand All @@ -341,12 +340,12 @@ async fn bundle_command(
if !flags.no_check {
// TODO(@kitsonk) support bundling for workers
let lib = if flags.unstable {
module_graph2::TypeLib::UnstableDenoWindow
module_graph::TypeLib::UnstableDenoWindow
} else {
module_graph2::TypeLib::DenoWindow
module_graph::TypeLib::DenoWindow
};
let graph = graph.clone();
let result_info = graph.check(module_graph2::CheckOptions {
let result_info = graph.check(module_graph::CheckOptions {
debug,
emit: false,
lib,
Expand All @@ -364,7 +363,7 @@ async fn bundle_command(
}

let (output, stats, maybe_ignored_options) =
graph.bundle(module_graph2::BundleOptions {
graph.bundle(module_graph::BundleOptions {
debug,
maybe_config_path: flags.config_path,
})?;
Expand Down Expand Up @@ -563,7 +562,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), AnyError> {
&program_state,
Permissions::allow_all(),
)?));
let mut builder = module_graph2::GraphBuilder2::new(
let mut builder = module_graph::GraphBuilder::new(
handler,
program_state.maybe_import_map.clone(),
program_state.lockfile.clone(),
Expand Down
Loading

0 comments on commit 6c743ed

Please sign in to comment.