feat(unstable): initial support for npm specifiers (denoland#15484)

Co-authored-by: Bartek Iwańczuk <[email protected]>

cli/Cargo.toml

@@ -83,6 +83,7 @@ node_resolver = "=0.1.1"
 notify = "=5.0.0-pre.15"
 once_cell = "=1.12.0"
 os_pipe = "=1.0.1"
+path-clean = "=0.1.0"
 percent-encoding = "=2.1.0"
 pin-project = "1.0.11" # don't pin because they yank crates from cargo
 rand = { version = "=0.8.5", features = ["small_rng"] }

cli/cache/mod.rs

@@ -2,7 +2,9 @@
 
 use crate::errors::get_error_class_name;
 use crate::file_fetcher::FileFetcher;
+use crate::npm;
 
+use deno_core::futures;
 use deno_core::futures::FutureExt;
 use deno_core::ModuleSpecifier;
 use deno_graph::source::CacheInfo;
@@ -53,6 +55,10 @@ impl FetchCacher {
 
 impl Loader for FetchCacher {
  fn get_cache_info(&self, specifier: &ModuleSpecifier) -> Option<CacheInfo> {
+ if specifier.scheme() == "npm" {
+ return None;
+ }
+
  let local = self.file_fetcher.get_local_path(specifier)?;
  if local.is_file() {
  let emit = self
@@ -74,6 +80,17 @@ impl Loader for FetchCacher {
  specifier: &ModuleSpecifier,
  is_dynamic: bool,
  ) -> LoadFuture {
+ if specifier.scheme() == "npm" {
+ return Box::pin(futures::future::ready(
+ match npm::NpmPackageReference::from_specifier(specifier) {
+ Ok(_) => Ok(Some(deno_graph::source::LoadResponse::External {
+ specifier: specifier.clone(),
+ })),
+ Err(err) => Err(err),
+ },
+ ));
+ }
+
  let specifier = specifier.clone();
  let mut permissions = if is_dynamic {
  self.dynamic_permissions.clone()

cli/compat/esm_resolver.rs

@@ -11,6 +11,7 @@ use deno_core::url::Url;
 use deno_core::ModuleSpecifier;
 use deno_graph::source::ResolveResponse;
 use deno_graph::source::Resolver;
+use deno_runtime::deno_node::DEFAULT_CONDITIONS;
 use regex::Regex;
 use std::path::PathBuf;
 
@@ -74,8 +75,6 @@ impl Resolver for NodeEsmResolver {
  }
 }
 
-static DEFAULT_CONDITIONS: &[&str] = &["deno", "node", "import"];
-
 /// This function is an implementation of `defaultResolve` in
 /// `lib/internal/modules/esm/resolve.js` from Node.
 fn node_resolve(

cli/compat/mod.rs

@@ -1,6 +1,6 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 
-mod errors;
+pub mod errors;
 mod esm_resolver;
 
 use crate::file_fetcher::FileFetcher;
@@ -75,15 +75,15 @@ static NODE_COMPAT_URL: Lazy<String> = Lazy::new(|| {
 static GLOBAL_URL_STR: Lazy<String> =
  Lazy::new(|| format!("{}node/global.ts", NODE_COMPAT_URL.as_str()));
 
-static PROCESS_URL_STR: Lazy<String> =
- Lazy::new(|| format!("{}node/process.ts", NODE_COMPAT_URL.as_str()));
-
 pub static GLOBAL_URL: Lazy<Url> =
  Lazy::new(|| Url::parse(&GLOBAL_URL_STR).unwrap());
 
 static MODULE_URL_STR: Lazy<String> =
  Lazy::new(|| format!("{}node/module.ts", NODE_COMPAT_URL.as_str()));
 
+pub static MODULE_ALL_URL: Lazy<Url> =
+ Lazy::new(|| Url::parse(&MODULE_ALL_URL_STR).unwrap());
+
 static MODULE_ALL_URL_STR: Lazy<String> =
  Lazy::new(|| format!("{}node/module_all.ts", NODE_COMPAT_URL.as_str()));
 
@@ -98,7 +98,7 @@ pub fn get_node_imports() -> Vec<(Url, Vec<String>)> {
  vec![(COMPAT_IMPORT_URL.clone(), vec![GLOBAL_URL_STR.clone()])]
 }
 
-fn try_resolve_builtin_module(specifier: &str) -> Option<Url> {
+pub fn try_resolve_builtin_module(specifier: &str) -> Option<Url> {
  if SUPPORTED_MODULES.contains(&specifier) {
  let ext = match specifier {
  "stream/promises" => "mjs",
@@ -112,28 +112,6 @@ fn try_resolve_builtin_module(specifier: &str) -> Option<Url> {
  }
 }
 
-#[allow(unused)]
-pub async fn load_builtin_node_modules(
- js_runtime: &mut JsRuntime,
-) -> Result<(), AnyError> {
- let source_code = &format!(
- r#"(async function loadBuiltinNodeModules(moduleAllUrl, processUrl) {{
- const [moduleAll, processModule] = await Promise.all([
- import(moduleAllUrl),
- import(processUrl)
- ]);
- Deno[Deno.internal].require.initializeCommonJs(moduleAll.default, processModule.default);
- }})('{}', '{}');"#,
- MODULE_ALL_URL_STR.as_str(),
- PROCESS_URL_STR.as_str(),
- );
-
- let value =
- js_runtime.execute_script(&located_script_name!(), source_code)?;
- js_runtime.resolve_value(value).await?;
- Ok(())
-}
-
 #[allow(unused)]
 pub fn load_cjs_module_from_ext_node(
  js_runtime: &mut JsRuntime,
@@ -214,7 +192,7 @@ pub fn setup_builtin_modules(
 /// For all discovered reexports the analysis will be performed recursively.
 ///
 /// If successful a source code for equivalent ES module is returned.
-pub async fn translate_cjs_to_esm(
+pub fn translate_cjs_to_esm(
  file_fetcher: &FileFetcher,
  specifier: &ModuleSpecifier,
  code: String,
@@ -271,7 +249,7 @@ pub async fn translate_cjs_to_esm(
  // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object,
  // but it might not be necessary here since our analysis is more detailed?
  source.push(format!(
- "export const {} = reexport{}.{};",
+ "export const {0} = Deno[Deno.internal].require.bindExport(reexport{1}.{2}, reexport{1});",
  export, idx, export
  ));
  }
@@ -294,7 +272,10 @@ pub async fn translate_cjs_to_esm(
  for export in analysis.exports.iter().filter(|e| e.as_str() != "default") {
  // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object,
  // but it might not be necessary here since our analysis is more detailed?
- source.push(format!("export const {} = mod.{};", export, export));
+ source.push(format!(
+ "export const {} = Deno[Deno.internal].require.bindExport(mod.{}, mod);",
+ export, export
+ ));
  }
 
  let translated_source = source.join("\n");

cli/graph_util.rs

@@ -3,6 +3,8 @@
 use crate::colors;
 use crate::emit::TsTypeLib;
 use crate::errors::get_error_class_name;
+use crate::npm::NpmPackageReference;
+use crate::npm::NpmPackageReq;
 
 use deno_ast::ParsedSource;
 use deno_core::error::custom_error;
@@ -58,6 +60,7 @@ pub enum ModuleEntry {
 #[derive(Debug, Default)]
 pub struct GraphData {
  modules: HashMap<ModuleSpecifier, ModuleEntry>,
+ npm_packages: HashSet<NpmPackageReq>,
  /// Map of first known referrer locations for each module. Used to enhance
  /// error messages.
  referrer_map: HashMap<ModuleSpecifier, Range>,
@@ -91,6 +94,13 @@ impl GraphData {
  if !reload && self.modules.contains_key(&specifier) {
  continue;
  }
+ if specifier.scheme() == "npm" {
+ // the loader enforces npm specifiers are valid, so it's ok to unwrap here
+ let reference =
+ NpmPackageReference::from_specifier(&specifier).unwrap();
+ self.npm_packages.insert(reference.req);
+ continue;
+ }
  if let Some(found) = graph.redirects.get(&specifier) {
  let module_entry = ModuleEntry::Redirect(found.clone());
  self.modules.insert(specifier.clone(), module_entry);
@@ -167,6 +177,11 @@ impl GraphData {
  self.modules.iter()
  }
 
+ /// Gets the unique npm package requirements from all the encountered graphs.
+ pub fn npm_package_reqs(&self) -> Vec<NpmPackageReq> {
+ self.npm_packages.iter().cloned().collect()
+ }
+
  /// Walk dependencies from `roots` and return every encountered specifier.
  /// Return `None` if any modules are not known.
  pub fn walk<'a>(
@@ -199,6 +214,10 @@ impl GraphData {
  }
  }
  while let Some(specifier) = visiting.pop_front() {
+ if NpmPackageReference::from_specifier(specifier).is_ok() {
+ continue; // skip analyzing npm specifiers
+ }
+
  let (specifier, entry) = match self.modules.get_key_value(specifier) {
  Some(pair) => pair,
  None => return None,
@@ -228,7 +247,13 @@ impl GraphData {
  }
  }
  }
- for (_, dep) in dependencies.iter().rev() {
+ for (dep_specifier, dep) in dependencies.iter().rev() {
+ // todo(dsherret): ideally there would be a way to skip external dependencies
+ // in the graph here rather than specifically npm package references
+ if NpmPackageReference::from_str(dep_specifier).is_ok() {
+ continue;
+ }
+
  if !dep.is_dynamic || follow_dynamic {
  let mut resolutions = vec![&dep.maybe_code];
  if check_types {
@@ -278,6 +303,7 @@ impl GraphData {
  }
  Some(Self {
  modules,
+ npm_packages: self.npm_packages.clone(),
  referrer_map,
  // TODO(nayeemrmn): Implement `Clone` on `GraphImport`.
  graph_imports: self

cli/lsp/cache.rs

@@ -67,7 +67,7 @@ impl CacheMetadata {
  &self,
  specifier: &ModuleSpecifier,
  ) -> Option<Arc<HashMap<MetadataKey, String>>> {
- if specifier.scheme() == "file" {
+ if specifier.scheme() == "file" || specifier.scheme() == "npm" {
  return None;
  }
  let version = self
@@ -83,7 +83,7 @@ impl CacheMetadata {
  }
 
  fn refresh(&self, specifier: &ModuleSpecifier) -> Option<Metadata> {
- if specifier.scheme() == "file" {
+ if specifier.scheme() == "file" || specifier.scheme() == "npm" {
  return None;
  }
  let cache_filename = self.cache.get_cache_filename(specifier)?;

cli/lsp/diagnostics.rs

@@ -14,6 +14,7 @@ use super::tsc::TsServer;
 
 use crate::args::LintConfig;
 use crate::diagnostics;
+use crate::npm::NpmPackageReference;
 
 use deno_ast::MediaType;
 use deno_core::anyhow::anyhow;
@@ -846,6 +847,8 @@ fn diagnose_resolved(
  .push(DenoDiagnostic::NoAssertType.to_lsp_diagnostic(&range)),
  }
  }
+ } else if NpmPackageReference::from_specifier(specifier).is_ok() {
+ // ignore npm specifiers for now
  } else {
  // When the document is not available, it means that it cannot be found
  // in the cache or locally on the disk, so we want to issue a diagnostic

cli/main.rs

@@ -23,7 +23,7 @@ mod lockfile;
 mod logger;
 mod lsp;
 mod module_loader;
-#[allow(unused)]
+mod node;
 mod npm;
 mod ops;
 mod proc_state;