feat(unstable): optional deno_modules directory (denoland#19977)

Closes denoland#15633

cli/args/config_file.rs

@@ -673,6 +673,7 @@ pub struct ConfigFileJson {
  pub lock: Option<Value>,
  pub exclude: Option<Value>,
  pub node_modules_dir: Option<bool>,
+ pub deno_modules_dir: Option<bool>,
 }
 
 #[derive(Clone, Debug)]
@@ -858,6 +859,26 @@ impl ConfigFile {
  self.json.node_modules_dir
  }
 
+ pub fn deno_modules_dir(&self) -> Option<bool> {
+ self.json.deno_modules_dir
+ }
+
+ pub fn deno_modules_dir_path(&self) -> Option<PathBuf> {
+ if self.json.deno_modules_dir == Some(true) {
+ Some(
+ self
+ .specifier
+ .to_file_path()
+ .unwrap()
+ .parent()
+ .unwrap()
+ .join("deno_modules"),
+ )
+ } else {
+ None
+ }
+ }
+
  pub fn to_import_map_value(&self) -> Value {
  let mut value = serde_json::Map::with_capacity(2);
  if let Some(imports) = &self.json.imports {
@@ -874,13 +895,17 @@ impl ConfigFile {
  }
 
  pub fn to_files_config(&self) -> Result<Option<FilesConfig>, AnyError> {
- let exclude: Vec<String> = if let Some(exclude) = self.json.exclude.clone()
- {
- serde_json::from_value(exclude)
- .context("Failed to parse \"exclude\" configuration")?
- } else {
- Vec::new()
- };
+ let mut exclude: Vec<String> =
+ if let Some(exclude) = self.json.exclude.clone() {
+ serde_json::from_value(exclude)
+ .context("Failed to parse \"exclude\" configuration")?
+ } else {
+ Vec::new()
+ };
+
+ if self.deno_modules_dir() == Some(true) {
+ exclude.push("deno_modules".to_string());
+ }
 
  let raw_files_config = SerializedFilesConfig {
  exclude,

cli/args/flags.rs

@@ -372,6 +372,7 @@ pub struct Flags {
  pub type_check_mode: TypeCheckMode,
  pub config_flag: ConfigFlag,
  pub node_modules_dir: Option<bool>,
+ pub deno_modules_dir: Option<bool>,
  pub enable_testing_features: bool,
  pub ext: Option<String>,
  pub ignore: Vec<PathBuf>,
@@ -1445,6 +1446,7 @@ TypeScript compiler cache: Subdirectory containing TS compiler output.",
  .arg(config_arg())
  .arg(import_map_arg())
  .arg(node_modules_dir_arg())
+ .arg(deno_modules_dir_arg())
  .arg(
  Arg::new("json")
  .long("json")
@@ -1988,6 +1990,7 @@ Remote modules and multiple modules may also be specified:
  .arg(import_map_arg())
  .arg(lock_arg())
  .arg(node_modules_dir_arg())
+ .arg(deno_modules_dir_arg())
  .arg(reload_arg())
  .arg(ca_file_arg()))
 }
@@ -2002,6 +2005,7 @@ fn compile_args_without_check_args(app: Command) -> Command {
  .arg(no_remote_arg())
  .arg(no_npm_arg())
  .arg(node_modules_dir_arg())
+ .arg(deno_modules_dir_arg())
  .arg(config_arg())
  .arg(no_config_arg())
  .arg(reload_arg())
@@ -2560,6 +2564,16 @@ fn node_modules_dir_arg() -> Arg {
  .help("Enables or disables the use of a local node_modules folder for npm packages")
 }
 
+fn deno_modules_dir_arg() -> Arg {
+ Arg::new("deno-modules-dir")
+ .long("deno-modules-dir")
+ .num_args(0..=1)
+ .value_parser(value_parser!(bool))
+ .default_missing_value("true")
+ .require_equals(true)
+ .help("UNSTABLE: Enables or disables the use of a local deno_modules folder for remote modules")
+}
+
 fn unsafely_ignore_certificate_errors_arg() -> Arg {
  Arg::new("unsafely-ignore-certificate-errors")
  .long("unsafely-ignore-certificate-errors")
@@ -2847,7 +2861,7 @@ fn info_parse(flags: &mut Flags, matches: &mut ArgMatches) {
  import_map_arg_parse(flags, matches);
  location_arg_parse(flags, matches);
  ca_file_arg_parse(flags, matches);
- node_modules_dir_arg_parse(flags, matches);
+ node_and_deno_modules_dir_arg_parse(flags, matches);
  lock_arg_parse(flags, matches);
  no_lock_arg_parse(flags, matches);
  no_remote_arg_parse(flags, matches);
@@ -3107,7 +3121,7 @@ fn vendor_parse(flags: &mut Flags, matches: &mut ArgMatches) {
  config_args_parse(flags, matches);
  import_map_arg_parse(flags, matches);
  lock_arg_parse(flags, matches);
- node_modules_dir_arg_parse(flags, matches);
+ node_and_deno_modules_dir_arg_parse(flags, matches);
  reload_arg_parse(flags, matches);
 
  flags.subcommand = DenoSubcommand::Vendor(VendorFlags {
@@ -3133,7 +3147,7 @@ fn compile_args_without_check_parse(
  import_map_arg_parse(flags, matches);
  no_remote_arg_parse(flags, matches);
  no_npm_arg_parse(flags, matches);
- node_modules_dir_arg_parse(flags, matches);
+ node_and_deno_modules_dir_arg_parse(flags, matches);
  config_args_parse(flags, matches);
  reload_arg_parse(flags, matches);
  lock_args_parse(flags, matches);
@@ -3387,8 +3401,12 @@ fn no_npm_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) {
  }
 }
 
-fn node_modules_dir_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) {
+fn node_and_deno_modules_dir_arg_parse(
+ flags: &mut Flags,
+ matches: &mut ArgMatches,
+) {
  flags.node_modules_dir = matches.remove_one::<bool>("node-modules-dir");
+ flags.deno_modules_dir = matches.remove_one::<bool>("deno-modules-dir");
 }
 
 fn reload_arg_validate(urlstr: &str) -> Result<String, String> {
@@ -5723,6 +5741,41 @@ mod tests {
  );
  }
 
+ #[test]
+ fn deno_modules_dir() {
+ let r =
+ flags_from_vec(svec!["deno", "run", "--deno-modules-dir", "script.ts"]);
+ assert_eq!(
+ r.unwrap(),
+ Flags {
+ subcommand: DenoSubcommand::Run(RunFlags {
+ script: "script.ts".to_string(),
+ watch: Default::default(),
+ }),
+ deno_modules_dir: Some(true),
+ ..Flags::default()
+ }
+ );
+
+ let r = flags_from_vec(svec![
+ "deno",
+ "run",
+ "--deno-modules-dir=false",
+ "script.ts"
+ ]);
+ assert_eq!(
+ r.unwrap(),
+ Flags {
+ subcommand: DenoSubcommand::Run(RunFlags {
+ script: "script.ts".to_string(),
+ watch: Default::default(),
+ }),
+ deno_modules_dir: Some(false),
+ ..Flags::default()
+ }
+ );
+ }
+
  #[test]
  fn cached_only() {
  let r = flags_from_vec(svec!["deno", "run", "--cached-only", "script.ts"]);

cli/args/mod.rs

@@ -539,6 +539,7 @@ pub struct CliOptions {
  flags: Flags,
  initial_cwd: PathBuf,
  maybe_node_modules_folder: Option<PathBuf>,
+ maybe_deno_modules_folder: Option<PathBuf>,
  maybe_config_file: Option<ConfigFile>,
  maybe_package_json: Option<PackageJson>,
  maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
@@ -567,13 +568,18 @@ impl CliOptions {
  eprintln!("{}", colors::yellow(msg));
  }
 
- let maybe_node_modules_folder = resolve_local_node_modules_folder(
+ let maybe_node_modules_folder = resolve_node_modules_folder(
  &initial_cwd,
  &flags,
  maybe_config_file.as_ref(),
  maybe_package_json.as_ref(),
  )
  .with_context(|| "Resolving node_modules folder.")?;
+ let maybe_deno_modules_folder = resolve_deno_modules_folder(
+ &initial_cwd,
+ &flags,
+ maybe_config_file.as_ref(),
+ );
 
  Ok(Self {
  flags,
@@ -582,6 +588,7 @@ impl CliOptions {
  maybe_lockfile,
  maybe_package_json,
  maybe_node_modules_folder,
+ maybe_deno_modules_folder,
  overrides: Default::default(),
  })
  }
@@ -865,6 +872,10 @@ impl CliOptions {
  .map(|path| ModuleSpecifier::from_directory_path(path).unwrap())
  }
 
+ pub fn deno_modules_dir_path(&self) -> Option<&PathBuf> {
+ self.maybe_deno_modules_folder.as_ref()
+ }
+
  pub fn resolve_root_cert_store_provider(
  &self,
  ) -> Arc<dyn RootCertStoreProvider> {
@@ -1159,7 +1170,7 @@ impl CliOptions {
 }
 
 /// Resolves the path to use for a local node_modules folder.
-fn resolve_local_node_modules_folder(
+fn resolve_node_modules_folder(
  cwd: &Path,
  flags: &Flags,
  maybe_config_file: Option<&ConfigFile>,
@@ -1188,6 +1199,31 @@ fn resolve_local_node_modules_folder(
  Ok(Some(canonicalize_path_maybe_not_exists(&path)?))
 }
 
+fn resolve_deno_modules_folder(
+ cwd: &Path,
+ flags: &Flags,
+ maybe_config_file: Option<&ConfigFile>,
+) -> Option<PathBuf> {
+ let use_deno_modules_dir = flags
+ .deno_modules_dir
+ .or_else(|| maybe_config_file.and_then(|c| c.deno_modules_dir()))
+ .unwrap_or(false);
+ // Unlike the node_modules directory, there is no need to canonicalize
+ // this directory because it's just used as a cache and the resolved
+ // specifier is not based on the canonicalized path (unlike the modules
+ // in the node_modules folder).
+ if !use_deno_modules_dir {
+ None
+ } else if let Some(config_path) = maybe_config_file
+ .as_ref()
+ .and_then(|c| c.specifier.to_file_path().ok())
+ {
+ Some(config_path.parent().unwrap().join("deno_modules"))
+ } else {
+ Some(cwd.join("deno_modules"))
+ }
+}
+
 fn resolve_import_map_specifier(
  maybe_import_map_path: Option<&str>,
  maybe_config_file: Option<&ConfigFile>,

cli/cache/deno_dir.rs

@@ -71,7 +71,6 @@ impl DenoDir {
  root,
  gen_cache: DiskCache::new(&gen_path),
  };
- deno_dir.gen_cache.ensure_dir_exists(&gen_path)?;
 
  Ok(deno_dir)
  }

cli/cache/disk_cache.rs

@@ -8,7 +8,6 @@ use deno_core::url::Host;
 use deno_core::url::Url;
 use std::ffi::OsStr;
 use std::fs;
-use std::io;
 use std::path::Component;
 use std::path::Path;
 use std::path::PathBuf;
@@ -20,13 +19,6 @@ pub struct DiskCache {
  pub location: PathBuf,
 }
 
-fn with_io_context<T: AsRef<str>>(
- e: &std::io::Error,
- context: T,
-) -> std::io::Error {
- std::io::Error::new(e.kind(), format!("{} (for '{}')", e, context.as_ref()))
-}
-
 impl DiskCache {
  /// `location` must be an absolute path.
  pub fn new(location: &Path) -> Self {
@@ -36,27 +28,6 @@ impl DiskCache {
  }
  }
 
- /// Ensures the location of the cache.
- pub fn ensure_dir_exists(&self, path: &Path) -> io::Result<()> {
- if path.is_dir() {
- return Ok(());
- }
- fs::create_dir_all(path).map_err(|e| {
- io::Error::new(
- e.kind(),
- format!(
- concat!(
- "Could not create TypeScript compiler cache location: {}\n",
- "Check the permission of the directory.\n",
- "{:#}",
- ),
- path.display(),
- e
- ),
- )
- })
- }
-
  fn get_cache_filename(&self, url: &Url) -> Option<PathBuf> {
  let mut out = PathBuf::new();
 
@@ -78,7 +49,7 @@ impl DiskCache {
  out.push(path_seg);
  }
  }
- "http" | "https" | "data" | "blob" => out = url_to_filename(url)?,
+ "http" | "https" | "data" | "blob" => out = url_to_filename(url).ok()?,
  "file" => {
  let path = match url.to_file_path() {
  Ok(path) => path,
@@ -149,12 +120,7 @@ impl DiskCache {
 
  pub fn set(&self, filename: &Path, data: &[u8]) -> std::io::Result<()> {
  let path = self.location.join(filename);
- match path.parent() {
- Some(parent) => self.ensure_dir_exists(parent),
- None => Ok(()),
- }?;
  atomic_write_file(&path, data, CACHE_PERM)
- .map_err(|e| with_io_context(&e, format!("{:#?}", &path)))
  }
 }
 
@@ -164,28 +130,13 @@ mod tests {
  use test_util::TempDir;
 
  #[test]
- fn test_create_cache_if_dir_exits() {
- let cache_location = TempDir::new();
- let cache_path = cache_location.path().join("foo");
- let cache = DiskCache::new(cache_path.as_path());
- cache
- .ensure_dir_exists(&cache.location)
- .expect("Testing expect:");
- assert!(cache_path.is_dir());
- }
-
- #[test]
- fn test_create_cache_if_dir_not_exits() {
+ fn test_set_get_cache_file() {
  let temp_dir = TempDir::new();
- let cache_location = temp_dir.path();
- cache_location.remove_dir_all();
- let cache_location = cache_location.join("foo");
- assert!(!cache_location.is_dir());
- let cache = DiskCache::new(cache_location.as_path());
- cache
- .ensure_dir_exists(&cache.location)
- .expect("Testing expect:");
- assert!(cache_location.is_dir());
+ let sub_dir = temp_dir.path().join("sub_dir");
+ let cache = DiskCache::new(&sub_dir.to_path_buf());
+ let path = PathBuf::from("foo/bar.txt");
+ cache.set(&path, b"hello").unwrap();
+ assert_eq!(cache.get(&path).unwrap(), b"hello");
  }
 
  #[test]