refactor: snapshotting of runtime/ and cli/ (denoland#21430)

This commit removes some of the technical debt related 
to snapshotting JS code:
- "cli/ops/mod.rs" and "cli/build.rs" no longer define "cli" extension
which was not required anymore
- Cargo features for "deno_runtime" crate have been unified in
"cli/Cargo.toml"
- "cli/build.rs" uses "deno_runtime::snapshot::create_runtime_snapshot"
API
instead of copy-pasting the code
- "cli/js/99_main.js" was completely removed as it's not necessary
anymore

Towards denoland#21137

cli/Cargo.toml

@@ -39,7 +39,10 @@ __runtime_js_sources = ["deno_runtime/__runtime_js_sources"]
 __vendored_zlib_ng = ["flate2/zlib-ng-compat", "libz-sys/zlib-ng"]
 
 [build-dependencies]
-deno_runtime = { workspace = true, features = ["exclude_runtime_main_js", "include_js_files_for_snapshotting"] }
+# TODO(bartlomieju): should we not include `dont_create_runtime_snapshot`
+# feature here and actually create the snapshot in `deno_runtime` build script?
+# How do we pass options?
+deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] }
 deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
 lazy-regex.workspace = true
 serde.workspace = true
@@ -63,7 +66,7 @@ deno_graph = "=0.61.5"
 deno_lint = { version = "=0.52.2", features = ["docs"] }
 deno_lockfile.workspace = true
 deno_npm = "0.15.2"
-deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "exclude_runtime_main_js", "include_js_files_for_snapshotting"] }
+deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] }
 deno_semver = "0.5.1"
 deno_task_shell = "=0.14.0"
 eszip = "=0.55.5"

cli/build.rs

@@ -4,8 +4,6 @@ use std::env;
 use std::path::PathBuf;
 
 use deno_core::snapshot_util::*;
-use deno_core::ExtensionFileSource;
-use deno_core::ExtensionFileSourceCode;
 use deno_runtime::*;
 
 mod ts {
@@ -318,36 +316,9 @@ mod ts {
  }
 }
 
-// Duplicated in `ops/mod.rs`. Keep in sync!
-deno_core::extension!(
- cli,
- deps = [runtime],
- esm_entry_point = "ext:cli/99_main.js",
- esm = [
- dir "js",
- "99_main.js"
- ],
- customizer = |ext: &mut deno_core::Extension| {
- ext.esm_files.to_mut().push(ExtensionFileSource {
- specifier: "ext:cli/runtime/js/99_main.js",
- code: ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
- deno_runtime::js::PATH_FOR_99_MAIN_JS,
- ),
- });
- }
-);
-
 #[cfg(not(feature = "__runtime_js_sources"))]
-#[must_use = "The files listed by create_cli_snapshot should be printed as 'cargo:rerun-if-changed' lines"]
-fn create_cli_snapshot(snapshot_path: PathBuf) -> CreateSnapshotOutput {
- use deno_core::Extension;
- use deno_runtime::deno_cache::SqliteBackedCache;
- use deno_runtime::deno_cron::local::LocalCronHandler;
- use deno_runtime::deno_http::DefaultHttpPropertyExtractor;
- use deno_runtime::deno_kv::sqlite::SqliteDbHandler;
+fn create_cli_snapshot(snapshot_path: PathBuf) {
  use deno_runtime::ops::bootstrap::SnapshotOptions;
- use deno_runtime::permissions::PermissionsContainer;
- use std::sync::Arc;
 
  // NOTE(bartlomieju): keep in sync with `cli/version.rs`.
  // Ideally we could deduplicate that code.
@@ -359,76 +330,17 @@ fn create_cli_snapshot(snapshot_path: PathBuf) -> CreateSnapshotOutput {
  }
  }
 
- // NOTE(bartlomieju): ordering is important here, keep it in sync with
- // `runtime/worker.rs`, `runtime/web_worker.rs` and `runtime/build.rs`!
- let fs = Arc::new(deno_fs::RealFs);
- let extensions: Vec<Extension> = vec![
- deno_webidl::deno_webidl::init_ops(),
- deno_console::deno_console::init_ops(),
- deno_url::deno_url::init_ops(),
- deno_web::deno_web::init_ops::<PermissionsContainer>(
- Default::default(),
- Default::default(),
- ),
- deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(Default::default()),
- deno_cache::deno_cache::init_ops::<SqliteBackedCache>(None),
- deno_websocket::deno_websocket::init_ops::<PermissionsContainer>(
- "".to_owned(),
- None,
- None,
- ),
- deno_webstorage::deno_webstorage::init_ops(None),
- deno_crypto::deno_crypto::init_ops(None),
- deno_broadcast_channel::deno_broadcast_channel::init_ops(
- deno_broadcast_channel::InMemoryBroadcastChannel::default(),
- ),
- deno_ffi::deno_ffi::init_ops::<PermissionsContainer>(),
- deno_net::deno_net::init_ops::<PermissionsContainer>(None, None),
- deno_tls::deno_tls::init_ops(),
- deno_kv::deno_kv::init_ops(SqliteDbHandler::<PermissionsContainer>::new(
- None, None,
- )),
- deno_cron::deno_cron::init_ops(LocalCronHandler::new()),
- deno_napi::deno_napi::init_ops::<PermissionsContainer>(),
- deno_http::deno_http::init_ops::<DefaultHttpPropertyExtractor>(),
- deno_io::deno_io::init_ops(Default::default()),
- deno_fs::deno_fs::init_ops::<PermissionsContainer>(fs.clone()),
- deno_node::deno_node::init_ops::<PermissionsContainer>(None, fs),
- deno_runtime::runtime::init_ops(),
- deno_runtime::ops::runtime::deno_runtime::init_ops(
- "deno:runtime".parse().unwrap(),
- ),
- deno_runtime::ops::worker_host::deno_worker_host::init_ops(
- Arc::new(|_| unreachable!("not used in snapshot.")),
- None,
- ),
- deno_runtime::ops::fs_events::deno_fs_events::init_ops(),
- deno_runtime::ops::os::deno_os::init_ops(Default::default()),
- deno_runtime::ops::permissions::deno_permissions::init_ops(),
- deno_runtime::ops::process::deno_process::init_ops(),
- deno_runtime::ops::signal::deno_signal::init_ops(),
- deno_runtime::ops::tty::deno_tty::init_ops(),
- deno_runtime::ops::http::deno_http_runtime::init_ops(),
- deno_runtime::ops::bootstrap::deno_bootstrap::init_ops(Some(
- SnapshotOptions {
- deno_version: deno_version(),
- ts_version: ts::version(),
- v8_version: deno_core::v8_version(),
- target: std::env::var("TARGET").unwrap(),
- },
- )),
- cli::init_ops_and_esm(), // NOTE: This needs to be init_ops_and_esm!
- ];
-
- create_snapshot(CreateSnapshotOptions {
- cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
+ let snapshot_options = SnapshotOptions {
+ deno_version: deno_version(),
+ ts_version: ts::version(),
+ v8_version: deno_core::v8_version(),
+ target: std::env::var("TARGET").unwrap(),
+ };
+
+ deno_runtime::snapshot::create_runtime_snapshot(
  snapshot_path,
- startup_snapshot: deno_runtime::js::deno_isolate_init(),
- extensions,
- compression_cb: None,
- with_runtime_cb: None,
- skip_op_registration: false,
- })
+ snapshot_options,
+ );
 }
 
 fn git_commit_hash() -> String {
@@ -539,10 +451,7 @@ fn main() {
  #[cfg(not(feature = "__runtime_js_sources"))]
  {
  let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin");
- let output = create_cli_snapshot(cli_snapshot_path);
- for path in output.files_loaded_during_snapshot {
- println!("cargo:rerun-if-changed={}", path.display())
- }
+ create_cli_snapshot(cli_snapshot_path);
  }
 
  #[cfg(target_os = "windows")]

cli/ops/mod.rs

@@ -1,35 +1,5 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
-use deno_core::Extension;
-
 pub mod bench;
 pub mod jupyter;
 pub mod testing;
-
-pub fn cli_exts() -> Vec<Extension> {
- vec![
- #[cfg(not(feature = "__runtime_js_sources"))]
- cli::init_ops(),
- #[cfg(feature = "__runtime_js_sources")]
- cli::init_ops_and_esm(),
- ]
-}
-
-// ESM parts duplicated in `../build.rs`. Keep in sync!
-deno_core::extension!(cli,
- deps = [runtime],
- esm_entry_point = "ext:cli/99_main.js",
- esm = [
- dir "js",
- "40_testing.js",
- "99_main.js"
- ],
- customizer = |ext: &mut deno_core::Extension| {
- ext.esm_files.to_mut().push(deno_core::ExtensionFileSource {
- specifier: "ext:cli/runtime/js/99_main.js",
- code: deno_core::ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
- deno_runtime::js::PATH_FOR_99_MAIN_JS,
- ),
- });
- },
-);

cli/worker.rs

@@ -52,7 +52,6 @@ use crate::args::StorageKeyResolver;
 use crate::emit::Emitter;
 use crate::errors;
 use crate::npm::CliNpmResolver;
-use crate::ops;
 use crate::tools;
 use crate::tools::coverage::CoverageCollector;
 use crate::tools::run::hmr::HmrRunner;
@@ -459,7 +458,7 @@ impl CliMainWorkerFactory {
  &self,
  main_module: ModuleSpecifier,
  permissions: PermissionsContainer,
- mut custom_extensions: Vec<Extension>,
+ custom_extensions: Vec<Extension>,
  stdio: deno_runtime::deno_io::Stdio,
  ) -> Result<CliMainWorker, AnyError> {
  let shared = &self.shared;
@@ -564,9 +563,6 @@ impl CliMainWorkerFactory {
  .join(checksum::gen(&[key.as_bytes()]))
  });
 
- let mut extensions = ops::cli_exts();
- extensions.append(&mut custom_extensions);
-
  // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out
  // list of enabled features.
  let feature_checker = shared.feature_checker.clone();
@@ -601,7 +597,7 @@ impl CliMainWorkerFactory {
  .maybe_binary_npm_command_name
  .clone(),
  },
- extensions,
+ extensions: custom_extensions,
  startup_snapshot: crate::js::deno_isolate_init(),
  create_params: None,
  unsafely_ignore_certificate_errors: shared
@@ -753,8 +749,6 @@ fn create_web_worker_callback(
  let create_web_worker_cb =
  create_web_worker_callback(shared.clone(), stdio.clone());
 
- let extensions = ops::cli_exts();
-
  let maybe_storage_key = shared
  .storage_key_resolver
  .resolve_storage_key(&args.main_module);
@@ -800,7 +794,7 @@ fn create_web_worker_callback(
  .maybe_binary_npm_command_name
  .clone(),
  },
- extensions,
+ extensions: vec![],
  startup_snapshot: crate::js::deno_isolate_init(),
  unsafely_ignore_certificate_errors: shared
  .options

runtime/build.rs

@@ -214,7 +214,7 @@ mod startup_snapshot {
 
  pub fn create_runtime_snapshot(snapshot_path: PathBuf) {
  // NOTE(bartlomieju): ordering is important here, keep it in sync with
- // `runtime/worker.rs`, `runtime/web_worker.rs` and `cli/build.rs`!
+ // `runtime/worker.rs`, `runtime/web_worker.rs` and `runtime/snapshot.rs`!
  let fs = std::sync::Arc::new(deno_fs::RealFs);
  let mut extensions: Vec<Extension> = vec![
  deno_webidl::deno_webidl::init_ops_and_esm(),

runtime/snapshot.rs

@@ -1,6 +1,7 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
 use crate::ops;
+use crate::ops::bootstrap::SnapshotOptions;
 use crate::shared::maybe_transpile_source;
 use crate::shared::runtime;
 use deno_cache::SqliteBackedCache;
@@ -183,9 +184,12 @@ impl deno_kv::sqlite::SqliteDbHandlerPermissions for Permissions {
  }
 }
 
-pub fn create_runtime_snapshot(snapshot_path: PathBuf) {
+pub fn create_runtime_snapshot(
+ snapshot_path: PathBuf,
+ snapshot_options: SnapshotOptions,
+) {
  // NOTE(bartlomieju): ordering is important here, keep it in sync with
- // `runtime/worker.rs`, `runtime/web_worker.rs` and `cli/build.rs`!
+ // `runtime/worker.rs`, `runtime/web_worker.rs` and `runtime/snapshot.rs`!
  let fs = std::sync::Arc::new(deno_fs::RealFs);
  let mut extensions: Vec<Extension> = vec![
  deno_webidl::deno_webidl::init_ops_and_esm(),
@@ -234,7 +238,7 @@ pub fn create_runtime_snapshot(snapshot_path: PathBuf) {
  ops::signal::deno_signal::init_ops(),
  ops::tty::deno_tty::init_ops(),
  ops::http::deno_http_runtime::init_ops(),
- ops::bootstrap::deno_bootstrap::init_ops(None),
+ ops::bootstrap::deno_bootstrap::init_ops(Some(snapshot_options)),
  ];
 
  for extension in &mut extensions {

runtime/web_worker.rs

@@ -397,7 +397,7 @@ impl WebWorker {
  });
 
  // NOTE(bartlomieju): ordering is important here, keep it in sync with
- // `runtime/build.rs`, `runtime/worker.rs` and `cli/build.rs`!
+ // `runtime/build.rs`, `runtime/worker.rs` and `runtime/snapshot.rs`!
 
  let mut extensions = vec![
  // Web APIs

runtime/worker.rs

@@ -297,7 +297,7 @@ impl MainWorker {
  });
 
  // NOTE(bartlomieju): ordering is important here, keep it in sync with
- // `runtime/build.rs`, `runtime/web_worker.rs` and `cli/build.rs`!
+ // `runtime/build.rs`, `runtime/web_worker.rs` and `runtime/snapshot.rs`!
  let mut extensions = vec![
  // Web APIs
  deno_webidl::deno_webidl::init_ops_and_esm(),