build: allow disabling snapshots for dev (denoland#20048)

Closes denoland#19399 (running without snapshots at all was suggested as an
alternative solution).

Adds a `__runtime_js_sources` pseudo-private feature to load extension
JS sources at runtime for faster development, instead of building and
loading snapshots or embedding sources in the binary. Will only work in
a development environment obviously.

Try running `cargo test --features __runtime_js_sources
integration::node_unit_tests::os_test`. Then break some behaviour in
`ext/node/polyfills/os.ts` e.g. make `function cpus() {}` return an
empty array, and run it again. Fix and then run again. No more build
time in between.

cli/Cargo.toml

@@ -26,6 +26,11 @@ name = "lsp_bench_standalone"
 harness = false
 path = "./bench/lsp_bench_standalone.rs"
 
+[features]
+# A dev feature to disable creations and loading of snapshots in favor of
+# loading JS sources at runtime.
+__runtime_js_sources = ["deno_runtime/__runtime_js_sources"]
+
 [build-dependencies]
 deno_runtime = { workspace = true, features = ["snapshot_from_snapshot", "include_js_files_for_snapshotting"] }
 deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }

cli/build.rs

@@ -2,16 +2,10 @@
 
 use std::env;
 use std::path::PathBuf;
-use std::sync::Arc;
 
 use deno_core::snapshot_util::*;
-use deno_core::Extension;
 use deno_core::ExtensionFileSource;
 use deno_core::ExtensionFileSourceCode;
-use deno_runtime::deno_cache::SqliteBackedCache;
-use deno_runtime::deno_http::DefaultHttpPropertyExtractor;
-use deno_runtime::deno_kv::sqlite::SqliteDbHandler;
-use deno_runtime::permissions::PermissionsContainer;
 use deno_runtime::*;
 
 mod ts {
@@ -304,12 +298,10 @@ mod ts {
  }
 }
 
-// FIXME(bartlomieju): information about which extensions were
-// already snapshotted is not preserved in the snapshot. This should be
-// fixed, so we can reliably depend on that information.
-// deps = [runtime]
+// Duplicated in `ops/mod.rs`. Keep in sync!
 deno_core::extension!(
  cli,
+ deps = [runtime],
  esm_entry_point = "ext:cli/99_main.js",
  esm = [
  dir "js",
@@ -326,8 +318,16 @@ deno_core::extension!(
  }
 );
 
+#[cfg(not(feature = "__runtime_js_sources"))]
 #[must_use = "The files listed by create_cli_snapshot should be printed as 'cargo:rerun-if-changed' lines"]
 fn create_cli_snapshot(snapshot_path: PathBuf) -> CreateSnapshotOutput {
+ use deno_core::Extension;
+ use deno_runtime::deno_cache::SqliteBackedCache;
+ use deno_runtime::deno_http::DefaultHttpPropertyExtractor;
+ use deno_runtime::deno_kv::sqlite::SqliteDbHandler;
+ use deno_runtime::permissions::PermissionsContainer;
+ use std::sync::Arc;
+
  // NOTE(bartlomieju): ordering is important here, keep it in sync with
  // `runtime/worker.rs`, `runtime/web_worker.rs` and `runtime/build.rs`!
  let fs = Arc::new(deno_fs::RealFs);
@@ -367,13 +367,14 @@ fn create_cli_snapshot(snapshot_path: PathBuf) -> CreateSnapshotOutput {
  deno_io::deno_io::init_ops(Default::default()),
  deno_fs::deno_fs::init_ops::<PermissionsContainer>(false, fs.clone()),
  deno_node::deno_node::init_ops::<PermissionsContainer>(None, fs),
+ deno_runtime::runtime::init_ops(),
  cli::init_ops_and_esm(), // NOTE: This needs to be init_ops_and_esm!
  ];
 
  create_snapshot(CreateSnapshotOptions {
  cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
  snapshot_path,
- startup_snapshot: Some(deno_runtime::js::deno_isolate_init()),
+ startup_snapshot: deno_runtime::js::deno_isolate_init(),
  extensions,
  compression_cb: None,
  with_runtime_cb: None,
@@ -483,10 +484,13 @@ fn main() {
  let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
  ts::create_compiler_snapshot(compiler_snapshot_path, &c);
 
- let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin");
- let output = create_cli_snapshot(cli_snapshot_path);
- for path in output.files_loaded_during_snapshot {
- println!("cargo:rerun-if-changed={}", path.display())
+ #[cfg(not(feature = "__runtime_js_sources"))]
+ {
+ let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin");
+ let output = create_cli_snapshot(cli_snapshot_path);
+ for path in output.files_loaded_during_snapshot {
+ println!("cargo:rerun-if-changed={}", path.display())
+ }
  }
 
  #[cfg(target_os = "windows")]

cli/js.rs

@@ -3,12 +3,20 @@
 use deno_core::Snapshot;
 use log::debug;
 
+#[cfg(not(feature = "__runtime_js_sources"))]
 static CLI_SNAPSHOT: &[u8] =
  include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
 
-pub fn deno_isolate_init() -> Snapshot {
+pub fn deno_isolate_init() -> Option<Snapshot> {
  debug!("Deno isolate init with snapshots.");
- Snapshot::Static(CLI_SNAPSHOT)
+ #[cfg(not(feature = "__runtime_js_sources"))]
+ {
+ Some(Snapshot::Static(CLI_SNAPSHOT))
+ }
+ #[cfg(feature = "__runtime_js_sources")]
+ {
+ None
+ }
 }
 
 #[cfg(test)]
@@ -18,7 +26,7 @@ mod tests {
  #[test]
  fn runtime_snapshot() {
  let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
- startup_snapshot: Some(deno_isolate_init()),
+ startup_snapshot: deno_isolate_init(),
  ..Default::default()
  });
  js_runtime

cli/ops/mod.rs

@@ -12,17 +12,38 @@ pub mod bench;
 pub mod testing;
 
 pub fn cli_exts(npm_resolver: Arc<CliNpmResolver>) -> Vec<Extension> {
- vec![deno_cli::init_ops(npm_resolver)]
+ vec![
+ #[cfg(not(feature = "__runtime_js_sources"))]
+ cli::init_ops(npm_resolver),
+ #[cfg(feature = "__runtime_js_sources")]
+ cli::init_ops_and_esm(npm_resolver),
+ ]
 }
 
-deno_core::extension!(deno_cli,
+// ESM parts duplicated in `../build.rs`. Keep in sync!
+deno_core::extension!(cli,
+ deps = [runtime],
  ops = [op_npm_process_state],
+ esm_entry_point = "ext:cli/99_main.js",
+ esm = [
+ dir "js",
+ "40_testing.js",
+ "99_main.js"
+ ],
  options = {
  npm_resolver: Arc<CliNpmResolver>,
  },
  state = |state, options| {
  state.put(options.npm_resolver);
  },
+ customizer = |ext: &mut deno_core::Extension| {
+ ext.esm_files.to_mut().push(deno_core::ExtensionFileSource {
+ specifier: "ext:cli/runtime/js/99_main.js",
+ code: deno_core::ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
+ deno_runtime::js::PATH_FOR_99_MAIN_JS,
+ ),
+ });
+ },
 );
 
 #[op]

cli/worker.rs

@@ -450,7 +450,7 @@ impl CliMainWorkerFactory {
  inspect: shared.options.is_inspecting,
  },
  extensions,
- startup_snapshot: Some(crate::js::deno_isolate_init()),
+ startup_snapshot: crate::js::deno_isolate_init(),
  create_params: None,
  unsafely_ignore_certificate_errors: shared
  .options
@@ -638,7 +638,7 @@ fn create_web_worker_callback(
  inspect: shared.options.is_inspecting,
  },
  extensions,
- startup_snapshot: Some(crate::js::deno_isolate_init()),
+ startup_snapshot: crate::js::deno_isolate_init(),
  unsafely_ignore_certificate_errors: shared
  .options
  .unsafely_ignore_certificate_errors
@@ -688,7 +688,7 @@ mod tests {
  let permissions = PermissionsContainer::new(Permissions::default());
 
  let options = WorkerOptions {
- startup_snapshot: Some(crate::js::deno_isolate_init()),
+ startup_snapshot: crate::js::deno_isolate_init(),
  ..Default::default()
  };
 

ext/node/build.rs

@@ -0,0 +1,7 @@
+// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+
+use std::env;
+
+fn main() {
+ println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
+}

ext/node/lib.rs

@@ -132,12 +132,7 @@ pub static NODE_ENV_VAR_ALLOWLIST: Lazy<HashSet<String>> = Lazy::new(|| {
 
 #[op]
 fn op_node_build_os() -> String {
- std::env::var("TARGET")
- .unwrap()
- .split('-')
- .nth(2)
- .unwrap()
- .to_string()
+ env!("TARGET").split('-').nth(2).unwrap().to_string()
 }
 
 #[op(fast)]

runtime/Cargo.toml

@@ -23,6 +23,9 @@ snapshot_from_snapshot = []
 include_js_files_for_snapshotting = [
  "deno_core/include_js_files_for_snapshotting",
 ]
+# A dev feature to disable creations and loading of snapshots in favor of
+# loading JS sources at runtime.
+__runtime_js_sources = ["dont_create_runtime_snapshot"]
 
 [lib]
 name = "deno_runtime"
@@ -64,6 +67,7 @@ winres.workspace = true
 winapi.workspace = true
 
 [dependencies]
+deno_ast.workspace = true
 deno_broadcast_channel.workspace = true
 deno_cache.workspace = true
 deno_console.workspace = true

runtime/build.rs

@@ -21,6 +21,7 @@ mod startup_snapshot {
  use deno_http::DefaultHttpPropertyExtractor;
  use std::path::Path;
 
+ // Duplicated in `worker.rs`. Keep in sync!
  fn maybe_transpile_source(
  source: &mut ExtensionFileSource,
  ) -> Result<(), AnyError> {
@@ -240,6 +241,7 @@ mod startup_snapshot {
  }
  }
 
+ // Duplicated in `worker.rs`. Keep in sync!
  deno_core::extension!(runtime,
  deps = [
  deno_webidl,

runtime/js.rs

@@ -1,17 +1,21 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
-#[cfg(not(feature = "dont_create_runtime_snapshot"))]
 use deno_core::Snapshot;
-#[cfg(not(feature = "dont_create_runtime_snapshot"))]
 use log::debug;
 
 #[cfg(not(feature = "dont_create_runtime_snapshot"))]
 static RUNTIME_SNAPSHOT: &[u8] =
  include_bytes!(concat!(env!("OUT_DIR"), "/RUNTIME_SNAPSHOT.bin"));
 
-#[cfg(not(feature = "dont_create_runtime_snapshot"))]
-pub fn deno_isolate_init() -> Snapshot {
+pub fn deno_isolate_init() -> Option<Snapshot> {
  debug!("Deno isolate init with snapshots.");
- Snapshot::Static(RUNTIME_SNAPSHOT)
+ #[cfg(not(feature = "dont_create_runtime_snapshot"))]
+ {
+ Some(Snapshot::Static(RUNTIME_SNAPSHOT))
+ }
+ #[cfg(feature = "dont_create_runtime_snapshot")]
+ {
+ None
+ }
 }
 
 #[cfg(not(feature = "include_js_files_for_snapshotting"))]

runtime/lib.rs

@@ -34,5 +34,6 @@ pub mod web_worker;
 pub mod worker;
 
 mod worker_bootstrap;
+pub use worker::runtime;
 pub use worker_bootstrap::BootstrapOptions;
 pub use worker_bootstrap::WorkerLogLevel;