refactor: integrate deno_graph into CLI (denoland#12369)

cli/Cargo.toml

@@ -41,8 +41,8 @@ winres = "0.1.11"
 [dependencies]
 deno_ast = { version = "0.2.0", features = ["bundler", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "typescript", "view", "visit"] }
 deno_core = { version = "0.102.0", path = "../core" }
-deno_doc = "0.14.0"
-deno_graph = "0.5.0"
+deno_doc = "0.15.0"
+deno_graph = "0.6.0"
 deno_lint = { version = "0.16.0", features = ["docs"] }
 deno_runtime = { version = "0.28.0", path = "../runtime" }
 deno_tls = { version = "0.7.0", path = "../ext/tls" }

cli/ast/bundle_hook.rs

@@ -4,6 +4,7 @@ use deno_ast::swc::bundler::ModuleRecord;
 use deno_ast::swc::common::Span;
 use deno_core::error::AnyError;
 
+/// This contains the logic for Deno to rewrite the `import.meta` when bundling.
 pub struct BundleHook;
 
 impl Hook for BundleHook {

cli/cache.rs

@@ -0,0 +1,349 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+use crate::disk_cache::DiskCache;
+use crate::file_fetcher::FileFetcher;
+
+use deno_core::error::AnyError;
+use deno_core::futures::future;
+use deno_core::futures::FutureExt;
+use deno_core::serde::Deserialize;
+use deno_core::serde::Serialize;
+use deno_core::serde_json;
+use deno_core::ModuleSpecifier;
+use deno_graph::source::CacheInfo;
+use deno_graph::source::LoadFuture;
+use deno_graph::source::LoadResponse;
+use deno_graph::source::Loader;
+use deno_runtime::permissions::Permissions;
+use std::collections::HashMap;
+use std::sync::Arc;
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct EmitMetadata {
+ pub version_hash: String,
+}
+
+pub(crate) enum CacheType {
+ Declaration,
+ Emit,
+ SourceMap,
+ TypeScriptBuildInfo,
+ Version,
+}
+
+/// A trait which provides a concise implementation to getting and setting
+/// values in a cache.
+pub(crate) trait Cacher {
+ /// Get a value from the cache.
+ fn get(
+ &self,
+ cache_type: CacheType,
+ specifier: &ModuleSpecifier,
+ ) -> Option<String>;
+ /// Set a value in the cache.
+ fn set(
+ &mut self,
+ cache_type: CacheType,
+ specifier: &ModuleSpecifier,
+ value: String,
+ ) -> Result<(), AnyError>;
+}
+
+/// Combines the cacher trait along with the deno_graph Loader trait to provide
+/// a single interface to be able to load and cache modules when building a
+/// graph.
+pub(crate) trait CacherLoader: Cacher + Loader {
+ fn as_cacher(&self) -> &dyn Cacher;
+ fn as_mut_loader(&mut self) -> &mut dyn Loader;
+ fn as_mut_cacher(&mut self) -> &mut dyn Cacher;
+}
+
+/// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides
+/// a concise interface to the DENO_DIR when building module graphs.
+pub(crate) struct FetchCacher {
+ disk_cache: DiskCache,
+ dynamic_permissions: Permissions,
+ file_fetcher: Arc<FileFetcher>,
+ root_permissions: Permissions,
+}
+
+impl FetchCacher {
+ pub fn new(
+ disk_cache: DiskCache,
+ file_fetcher: FileFetcher,
+ root_permissions: Permissions,
+ dynamic_permissions: Permissions,
+ ) -> Self {
+ let file_fetcher = Arc::new(file_fetcher);
+
+ Self {
+ disk_cache,
+ dynamic_permissions,
+ file_fetcher,
+ root_permissions,
+ }
+ }
+
+ fn get_emit_metadata(
+ &self,
+ specifier: &ModuleSpecifier,
+ ) -> Option<EmitMetadata> {
+ let filename = self
+ .disk_cache
+ .get_cache_filename_with_extension(specifier, "meta")?;
+ let bytes = self.disk_cache.get(&filename).ok()?;
+ serde_json::from_slice(&bytes).ok()
+ }
+
+ fn set_emit_metadata(
+ &self,
+ specifier: &ModuleSpecifier,
+ data: EmitMetadata,
+ ) -> Result<(), AnyError> {
+ let filename = self
+ .disk_cache
+ .get_cache_filename_with_extension(specifier, "meta")
+ .unwrap();
+ let bytes = serde_json::to_vec(&data)?;
+ self.disk_cache.set(&filename, &bytes).map_err(|e| e.into())
+ }
+}
+
+impl Loader for FetchCacher {
+ fn get_cache_info(&self, specifier: &ModuleSpecifier) -> Option<CacheInfo> {
+ let local = self.file_fetcher.get_local_path(specifier)?;
+ if local.is_file() {
+ let location = &self.disk_cache.location;
+ let emit = self
+ .disk_cache
+ .get_cache_filename_with_extension(specifier, "js")
+ .map(|p| location.join(p))
+ .filter(|p| p.is_file());
+ let map = self
+ .disk_cache
+ .get_cache_filename_with_extension(specifier, "js.map")
+ .map(|p| location.join(p))
+ .filter(|p| p.is_file());
+ Some(CacheInfo {
+ local: Some(local),
+ emit,
+ map,
+ })
+ } else {
+ None
+ }
+ }
+
+ fn load(
+ &mut self,
+ specifier: &ModuleSpecifier,
+ is_dynamic: bool,
+ ) -> LoadFuture {
+ let specifier = specifier.clone();
+ let mut permissions = if is_dynamic {
+ self.dynamic_permissions.clone()
+ } else {
+ self.root_permissions.clone()
+ };
+ let file_fetcher = self.file_fetcher.clone();
+
+ async move {
+ let load_result = file_fetcher
+ .fetch(&specifier, &mut permissions)
+ .await
+ .map_or_else(
+ |err| {
+ if let Some(err) = err.downcast_ref::<std::io::Error>() {
+ if err.kind() == std::io::ErrorKind::NotFound {
+ return Ok(None);
+ }
+ }
+ Err(err)
+ },
+ |file| {
+ Ok(Some(LoadResponse {
+ specifier: file.specifier,
+ maybe_headers: file.maybe_headers,
+ content: file.source,
+ }))
+ },
+ );
+
+ (specifier, load_result)
+ }
+ .boxed()
+ }
+}
+
+impl Cacher for FetchCacher {
+ fn get(
+ &self,
+ cache_type: CacheType,
+ specifier: &ModuleSpecifier,
+ ) -> Option<String> {
+ let extension = match cache_type {
+ CacheType::Declaration => "d.ts",
+ CacheType::Emit => "js",
+ CacheType::SourceMap => "js.map",
+ CacheType::TypeScriptBuildInfo => "buildinfo",
+ CacheType::Version => {
+ return self.get_emit_metadata(specifier).map(|d| d.version_hash)
+ }
+ };
+ let filename = self
+ .disk_cache
+ .get_cache_filename_with_extension(specifier, extension)?;
+ self
+ .disk_cache
+ .get(&filename)
+ .ok()
+ .map(|b| String::from_utf8(b).ok())
+ .flatten()
+ }
+
+ fn set(
+ &mut self,
+ cache_type: CacheType,
+ specifier: &ModuleSpecifier,
+ value: String,
+ ) -> Result<(), AnyError> {
+ let extension = match cache_type {
+ CacheType::Declaration => "d.ts",
+ CacheType::Emit => "js",
+ CacheType::SourceMap => "js.map",
+ CacheType::TypeScriptBuildInfo => "buildinfo",
+ CacheType::Version => {
+ let data = if let Some(mut data) = self.get_emit_metadata(specifier) {
+ data.version_hash = value;
+ data
+ } else {
+ EmitMetadata {
+ version_hash: value,
+ }
+ };
+ return self.set_emit_metadata(specifier, data);
+ }
+ };
+ let filename = self
+ .disk_cache
+ .get_cache_filename_with_extension(specifier, extension)
+ .unwrap();
+ self
+ .disk_cache
+ .set(&filename, value.as_bytes())
+ .map_err(|e| e.into())
+ }
+}
+
+impl CacherLoader for FetchCacher {
+ fn as_cacher(&self) -> &dyn Cacher {
+ self
+ }
+
+ fn as_mut_loader(&mut self) -> &mut dyn Loader {
+ self
+ }
+
+ fn as_mut_cacher(&mut self) -> &mut dyn Cacher {
+ self
+ }
+}
+
+/// An in memory cache that is used by the runtime `Deno.emit()` API to provide
+/// the same behavior as the disk cache when sources are provided.
+#[derive(Debug)]
+pub(crate) struct MemoryCacher {
+ sources: HashMap<String, Arc<String>>,
+ declarations: HashMap<ModuleSpecifier, String>,
+ emits: HashMap<ModuleSpecifier, String>,
+ maps: HashMap<ModuleSpecifier, String>,
+ build_infos: HashMap<ModuleSpecifier, String>,
+ versions: HashMap<ModuleSpecifier, String>,
+}
+
+impl MemoryCacher {
+ pub fn new(sources: HashMap<String, Arc<String>>) -> Self {
+ Self {
+ sources,
+ declarations: HashMap::default(),
+ emits: HashMap::default(),
+ maps: HashMap::default(),
+ build_infos: HashMap::default(),
+ versions: HashMap::default(),
+ }
+ }
+}
+
+impl Loader for MemoryCacher {
+ fn load(
+ &mut self,
+ specifier: &ModuleSpecifier,
+ _is_dynamic: bool,
+ ) -> LoadFuture {
+ let mut specifier_str = specifier.to_string();
+ if !self.sources.contains_key(&specifier_str) {
+ specifier_str = specifier_str.replace("file:https:///", "/");
+ if !self.sources.contains_key(&specifier_str) {
+ specifier_str = specifier_str[3..].to_string();
+ }
+ }
+ let response = self.sources.get(&specifier_str).map(|c| LoadResponse {
+ specifier: specifier.clone(),
+ maybe_headers: None,
+ content: c.to_owned(),
+ });
+ Box::pin(future::ready((specifier.clone(), Ok(response))))
+ }
+}
+
+impl Cacher for MemoryCacher {
+ fn get(
+ &self,
+ cache_type: CacheType,
+ specifier: &ModuleSpecifier,
+ ) -> Option<String> {
+ match cache_type {
+ CacheType::Declaration => self.declarations.get(specifier).cloned(),
+ CacheType::Emit => self.emits.get(specifier).cloned(),
+ CacheType::SourceMap => self.maps.get(specifier).cloned(),
+ CacheType::TypeScriptBuildInfo => {
+ self.build_infos.get(specifier).cloned()
+ }
+ CacheType::Version => self.versions.get(specifier).cloned(),
+ }
+ }
+
+ fn set(
+ &mut self,
+ cache_type: CacheType,
+ specifier: &ModuleSpecifier,
+ value: String,
+ ) -> Result<(), AnyError> {
+ match cache_type {
+ CacheType::Declaration => {
+ self.declarations.insert(specifier.clone(), value)
+ }
+ CacheType::Emit => self.emits.insert(specifier.clone(), value),
+ CacheType::SourceMap => self.maps.insert(specifier.clone(), value),
+ CacheType::TypeScriptBuildInfo => {
+ self.build_infos.insert(specifier.clone(), value)
+ }
+ CacheType::Version => self.versions.insert(specifier.clone(), value),
+ };
+ Ok(())
+ }
+}
+
+impl CacherLoader for MemoryCacher {
+ fn as_cacher(&self) -> &dyn Cacher {
+ self
+ }
+
+ fn as_mut_loader(&mut self) -> &mut dyn Loader {
+ self
+ }
+
+ fn as_mut_cacher(&mut self) -> &mut dyn Cacher {
+ self
+ }
+}

cli/compat.rs

@@ -4,6 +4,7 @@ use deno_core::url::Url;
 use std::collections::HashMap;
 
 static STD_NODE: &str = "https://deno.land/std/node/";
+static GLOBAL_MODULE: &str = "global.ts";
 
 static SUPPORTED_MODULES: &[&str] = &[
  "assert",
@@ -50,8 +51,15 @@ static SUPPORTED_MODULES: &[&str] = &[
  "zlib",
 ];
 
-pub fn get_node_globals_url() -> Url {
- Url::parse(&format!("{}global.ts", STD_NODE)).unwrap()
+lazy_static::lazy_static! {
+ static ref GLOBAL_URL_STR: String = format!("{}{}", STD_NODE, GLOBAL_MODULE);
+ pub(crate) static ref GLOBAL_URL: Url = Url::parse(&GLOBAL_URL_STR).unwrap();
+ static ref COMPAT_IMPORT_URL: Url = Url::parse("flags:compat").unwrap();
+}
+
+/// Provide imports into a module graph when the compat flag is true.
+pub(crate) fn get_node_imports() -> Vec<(Url, Vec<String>)> {
+ vec![(COMPAT_IMPORT_URL.clone(), vec![GLOBAL_URL_STR.clone()])]
 }
 
 /// Create a map that can be used to update import map.