feat(vendor): support modifying remote files in vendor folder without…

… checksum errors (denoland#23979)

Includes:

* denoland/deno_graph#486
* denoland/deno_graph#488
* denoland/deno_lockfile#25
* denoland/deno_lockfile#22
* denoland/deno_graph#483
* denoland/deno_graph#470

Cargo.toml

@@ -47,7 +47,7 @@ deno_ast = { version = "=0.38.2", features = ["transpiling"] }
 deno_core = { version = "0.283.0" }
 
 deno_bench_util = { version = "0.147.0", path = "./bench_util" }
-deno_lockfile = "0.19.0"
+deno_lockfile = "0.20.0"
 deno_media_type = { version = "0.1.4", features = ["module_specifier"] }
 deno_permissions = { version = "0.13.0", path = "./runtime/permissions" }
 deno_runtime = { version = "0.161.0", path = "./runtime" }
@@ -100,7 +100,7 @@ chrono = { version = "0.4", default-features = false, features = ["std", "serde"
 console_static_text = "=0.8.1"
 data-encoding = "2.3.3"
 data-url = "=0.3.0"
-deno_cache_dir = "=0.7.1"
+deno_cache_dir = "=0.10.0"
 dlopen2 = "0.6.1"
 ecb = "=0.1.2"
 elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem"] }

cli/Cargo.toml

@@ -67,17 +67,17 @@ deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposa
 deno_cache_dir = { workspace = true }
 deno_config = "=0.16.3"
 deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
-deno_doc = { version = "=0.135.0", features = ["html", "syntect"] }
-deno_emit = "=0.40.3"
-deno_graph = { version = "=0.75.2", features = ["tokio_executor"] }
+deno_doc = { version = "=0.137.0", features = ["html", "syntect"] }
+deno_emit = "=0.41.0"
+deno_graph = { version = "=0.77.0", features = ["tokio_executor"] }
 deno_lint = { version = "=0.58.4", features = ["docs"] }
 deno_lockfile.workspace = true
-deno_npm = "=0.20.2"
+deno_npm = "=0.21.0"
 deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] }
 deno_semver = "=0.5.4"
 deno_task_shell = "=0.16.1"
 deno_terminal.workspace = true
-eszip = "=0.69.0"
+eszip = "=0.70.1"
 napi_sym.workspace = true
 
 async-trait.workspace = true

cli/args/lockfile.rs

@@ -2,18 +2,20 @@
 
 use std::path::PathBuf;
 
+use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_runtime::deno_node::PackageJson;
 
 use crate::args::ConfigFile;
+use crate::cache;
+use crate::util::fs::atomic_write_file;
 use crate::Flags;
 
 use super::DenoSubcommand;
 use super::InstallFlags;
 use super::InstallKind;
 
 pub use deno_lockfile::Lockfile;
-pub use deno_lockfile::LockfileError;
 
 pub fn discover(
  flags: &Flags,
@@ -54,6 +56,34 @@ pub fn discover(
  },
  };
 
- let lockfile = Lockfile::new(filename, flags.lock_write)?;
+ let lockfile = if flags.lock_write {
+ Lockfile::new_empty(filename, true)
+ } else {
+ read_lockfile_at_path(filename)?
+ };
  Ok(Some(lockfile))
 }
+
+pub fn read_lockfile_at_path(filename: PathBuf) -> Result<Lockfile, AnyError> {
+ match std::fs::read_to_string(&filename) {
+ Ok(text) => Ok(Lockfile::with_lockfile_content(filename, &text, false)?),
+ Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+ Ok(Lockfile::new_empty(filename, false))
+ }
+ Err(err) => Err(err).with_context(|| {
+ format!("Failed reading lockfile '{}'", filename.display())
+ }),
+ }
+}
+
+pub fn write_lockfile_if_has_changes(
+ lockfile: &Lockfile,
+) -> Result<(), AnyError> {
+ let Some(bytes) = lockfile.resolve_write_bytes() else {
+ return Ok(()); // nothing to do
+ };
+ // do an atomic write to reduce the chance of multiple deno
+ // processes corrupting the file
+ atomic_write_file(&lockfile.filename, bytes, cache::CACHE_PERM)
+ .context("Failed writing lockfile.")
+}

cli/args/mod.rs

@@ -34,8 +34,9 @@ pub use deno_config::TsConfigType;
 pub use deno_config::TsTypeLib;
 pub use deno_config::WorkspaceConfig;
 pub use flags::*;
+pub use lockfile::read_lockfile_at_path;
+pub use lockfile::write_lockfile_if_has_changes;
 pub use lockfile::Lockfile;
-pub use lockfile::LockfileError;
 pub use package_json::PackageJsonDepsProvider;
 
 use deno_ast::ModuleSpecifier;

cli/cache/mod.rs

@@ -228,7 +228,7 @@ impl Loader for FetchCacher {
  LoaderCacheSetting::Reload => {
  if matches!(file_fetcher.cache_setting(), CacheSetting::Only) {
  return Err(deno_core::anyhow::anyhow!(
- "Failed to resolve version constraint. Try running again without --cached-only"
+ "Could not resolve version constraint using only cached data. Try running again without --cached-only"
  ));
  }
  Some(CacheSetting::ReloadAll)

cli/cache/module_info.rs

@@ -150,8 +150,9 @@ pub struct ModuleInfoCacheModuleAnalyzer<'a> {
  parser: &'a dyn ModuleParser,
 }
 
+#[async_trait::async_trait(?Send)]
 impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
- fn analyze(
+ async fn analyze(
  &self,
  specifier: &ModuleSpecifier,
  source: Arc<str>,
@@ -176,8 +177,9 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
  }
 
  // otherwise, get the module info from the parsed source cache
+ // todo(23858): take advantage of this being async
  let analyzer = ParserModuleAnalyzer::new(self.parser);
- let module_info = analyzer.analyze(specifier, source, media_type)?;
+ let module_info = analyzer.analyze(specifier, source, media_type).await?;
 
  // then attempt to cache it
  if let Err(err) = self.module_info_cache.set_module_info(

cli/errors.rs

@@ -14,6 +14,7 @@ use deno_core::error::AnyError;
 use deno_graph::source::ResolveError;
 use deno_graph::ModuleError;
 use deno_graph::ModuleGraphError;
+use deno_graph::ModuleLoadError;
 use deno_graph::ResolutionError;
 use import_map::ImportMapError;
 use std::fmt::Write;
@@ -27,24 +28,56 @@ fn get_diagnostic_class(_: &ParseDiagnostic) -> &'static str {
 }
 
 fn get_module_graph_error_class(err: &ModuleGraphError) -> &'static str {
+ use deno_graph::JsrLoadError;
+ use deno_graph::NpmLoadError;
+ use deno_graph::WorkspaceLoadError;
+
  match err {
+ ModuleGraphError::ResolutionError(err)
+ | ModuleGraphError::TypesResolutionError(err) => {
+ get_resolution_error_class(err)
+ }
  ModuleGraphError::ModuleError(err) => match err {
- ModuleError::LoadingErr(_, _, err) => get_error_class_name(err.as_ref()),
  ModuleError::InvalidTypeAssertion { .. } => "SyntaxError",
  ModuleError::ParseErr(_, diagnostic) => get_diagnostic_class(diagnostic),
  ModuleError::UnsupportedMediaType { .. }
  | ModuleError::UnsupportedImportAttributeType { .. } => "TypeError",
- ModuleError::Missing(_, _)
- | ModuleError::MissingDynamic(_, _)
- | ModuleError::MissingWorkspaceMemberExports { .. }
- | ModuleError::UnknownExport { .. }
- | ModuleError::UnknownPackage { .. }
- | ModuleError::UnknownPackageReq { .. } => "NotFound",
+ ModuleError::Missing(_, _) | ModuleError::MissingDynamic(_, _) => {
+ "NotFound"
+ }
+ ModuleError::LoadingErr(_, _, err) => match err {
+ ModuleLoadError::Loader(err) => get_error_class_name(err.as_ref()),
+ ModuleLoadError::HttpsChecksumIntegrity(_)
+ | ModuleLoadError::TooManyRedirects => "Error",
+ ModuleLoadError::NodeUnknownBuiltinModule(_) => "NotFound",
+ ModuleLoadError::Decode(_) => "TypeError",
+ ModuleLoadError::Npm(err) => match err {
+ NpmLoadError::NotSupportedEnvironment
+ | NpmLoadError::PackageReqResolution(_)
+ | NpmLoadError::RegistryInfo(_) => "Error",
+ NpmLoadError::PackageReqReferenceParse(_) => "TypeError",
+ },
+ ModuleLoadError::Jsr(err) => match err {
+ JsrLoadError::UnsupportedManifestChecksum
+ | JsrLoadError::PackageFormat(_) => "TypeError",
+ JsrLoadError::ContentLoadExternalSpecifier
+ | JsrLoadError::ContentLoad(_)
+ | JsrLoadError::ContentChecksumIntegrity(_)
+ | JsrLoadError::PackageManifestLoad(_, _)
+ | JsrLoadError::PackageVersionManifestChecksumIntegrity(..)
+ | JsrLoadError::PackageVersionManifestLoad(_, _)
+ | JsrLoadError::RedirectInPackage(_) => "Error",
+ JsrLoadError::PackageNotFound(_)
+ | JsrLoadError::PackageReqNotFound(_)
+ | JsrLoadError::PackageVersionNotFound(_)
+ | JsrLoadError::UnknownExport { .. } => "NotFound",
+ },
+ ModuleLoadError::Workspace(err) => match err {
+ WorkspaceLoadError::MemberInvalidExportPath { .. } => "TypeError",
+ WorkspaceLoadError::MissingMemberExports { .. } => "NotFound",
+ },
+ },
  },
- ModuleGraphError::ResolutionError(err)
- | ModuleGraphError::TypesResolutionError(err) => {
- get_resolution_error_class(err)
- }
  }
 }
 

cli/factory.rs

@@ -665,7 +665,6 @@ impl CliFactory {
  self.options.clone(),
  self.npm_resolver().await?.clone(),
  self.module_graph_builder().await?.clone(),
- self.maybe_lockfile().clone(),
  self.type_checker().await?.clone(),
  )))
  })

cli/file_fetcher.rs

@@ -253,15 +253,30 @@ impl FileFetcher {
  deno_core::resolve_import(redirect_to, specifier.as_str())?;
  return Ok(Some(FileOrRedirect::Redirect(redirect)));
  }
- let Some(bytes) = self.http_cache.read_file_bytes(
+ let result = self.http_cache.read_file_bytes(
  &cache_key,
  maybe_checksum
  .as_ref()
  .map(|c| deno_cache_dir::Checksum::new(c.as_str())),
  deno_cache_dir::GlobalToLocalCopy::Allow,
- )?
- else {
- return Ok(None);
+ );
+ let bytes = match result {
+ Ok(Some(bytes)) => bytes,
+ Ok(None) => return Ok(None),
+ Err(err) => match err {
+ deno_cache_dir::CacheReadFileError::Io(err) => return Err(err.into()),
+ deno_cache_dir::CacheReadFileError::ChecksumIntegrity(err) => {
+ // convert to the equivalent deno_graph error so that it
+ // enhances it if this is passed to deno_graph
+ return Err(
+ deno_graph::source::ChecksumIntegrityError {
+ actual: err.actual,
+ expected: err.expected,
+ }
+ .into(),
+ );
+ }
+ },
  };
 
  Ok(Some(FileOrRedirect::File(File {