refactor(lsp): collect npm reqs by scope (denoland#24172)

cli/lsp/config.rs

@@ -778,13 +778,9 @@ impl Settings {
  specifier: &ModuleSpecifier,
  ) -> (&WorkspaceSettings, Option<&ModuleSpecifier>) {
  let Ok(path) = specifier_to_file_path(specifier) else {
- return (&self.unscoped, None);
+ return (&self.unscoped, self.first_folder.as_ref());
  };
  for (folder_uri, settings) in self.by_workspace_folder.iter().rev() {
- let mut settings = settings.as_ref();
- if self.first_folder.as_ref() == Some(folder_uri) {
- settings = settings.or(Some(&self.unscoped));
- }
  if let Some(settings) = settings {
  let Ok(folder_path) = specifier_to_file_path(folder_uri) else {
  continue;
@@ -794,7 +790,7 @@ impl Settings {
  }
  }
  }
- (&self.unscoped, None)
+ (&self.unscoped, self.first_folder.as_ref())
  }
 
  pub fn enable_settings_hash(&self) -> u64 {
@@ -1572,6 +1568,10 @@ pub struct ConfigTree {
 }
 
 impl ConfigTree {
+ pub fn root_scope(&self) -> Option<&ModuleSpecifier> {
+ self.first_folder.as_ref()
+ }
+
  pub fn root_data(&self) -> Option<&ConfigData> {
  self.first_folder.as_ref().and_then(|s| self.scopes.get(s))
  }
@@ -1583,10 +1583,6 @@ impl ConfigTree {
  .unwrap_or_default()
  }
 
- pub fn root_lockfile(&self) -> Option<&Arc<Mutex<Lockfile>>> {
- self.root_data().and_then(|d| d.lockfile.as_ref())
- }
-
  pub fn root_import_map(&self) -> Option<&Arc<ImportMap>> {
  self.root_data().and_then(|d| d.import_map.as_ref())
  }

cli/lsp/documents.rs

@@ -34,6 +34,7 @@ use deno_semver::npm::NpmPackageReqReference;
 use deno_semver::package::PackageReq;
 use indexmap::IndexMap;
 use std::borrow::Cow;
+use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::collections::HashSet;
@@ -903,7 +904,8 @@ pub struct Documents {
  /// settings.
  resolver: Arc<LspResolver>,
  /// The npm package requirements found in npm specifiers.
- npm_specifier_reqs: Arc<Vec<PackageReq>>,
+ npm_reqs_by_scope:
+ Arc<BTreeMap<Option<ModuleSpecifier>, BTreeSet<PackageReq>>>,
  /// Gets if any document had a node: specifier such that a @types/node package
  /// should be injected.
  has_injected_types_node_package: bool,
@@ -1093,10 +1095,11 @@ impl Documents {
  false
  }
 
- /// Returns a collection of npm package requirements.
- pub fn npm_package_reqs(&mut self) -> Arc<Vec<PackageReq>> {
+ pub fn npm_reqs_by_scope(
+ &mut self,
+ ) -> Arc<BTreeMap<Option<ModuleSpecifier>, BTreeSet<PackageReq>>> {
  self.calculate_npm_reqs_if_dirty();
- self.npm_specifier_reqs.clone()
+ self.npm_reqs_by_scope.clone()
  }
 
  /// Returns if a @types/node package was injected into the npm
@@ -1322,31 +1325,36 @@ impl Documents {
  /// document and the value is a set of specifiers that depend on that
  /// document.
  fn calculate_npm_reqs_if_dirty(&mut self) {
- let mut npm_reqs = HashSet::new();
- let mut has_node_builtin_specifier = false;
+ let mut npm_reqs_by_scope: BTreeMap<_, BTreeSet<_>> = Default::default();
+ let mut scopes_with_node_builtin_specifier = HashSet::new();
  let is_fs_docs_dirty = self.file_system_docs.set_dirty(false);
  if !is_fs_docs_dirty && !self.dirty {
  return;
  }
  let mut visit_doc = |doc: &Arc<Document>| {
+ let scope = doc
+ .file_referrer()
+ .and_then(|r| self.config.tree.scope_for_specifier(r))
+ .or(self.config.tree.root_scope());
+ let reqs = npm_reqs_by_scope.entry(scope.cloned()).or_default();
  for dependency in doc.dependencies().values() {
  if let Some(dep) = dependency.get_code() {
  if dep.scheme() == "node" {
- has_node_builtin_specifier = true;
+ scopes_with_node_builtin_specifier.insert(scope.cloned());
  }
  if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) {
- npm_reqs.insert(reference.into_inner().req);
+ reqs.insert(reference.into_inner().req);
  }
  }
  if let Some(dep) = dependency.get_type() {
  if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) {
- npm_reqs.insert(reference.into_inner().req);
+ reqs.insert(reference.into_inner().req);
  }
  }
  }
  if let Some(dep) = doc.maybe_types_dependency().maybe_specifier() {
  if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) {
- npm_reqs.insert(reference.into_inner().req);
+ reqs.insert(reference.into_inner().req);
  }
  }
  };
@@ -1358,12 +1366,21 @@ impl Documents {
  }
 
  // fill the reqs from the lockfile
- if let Some(lockfile) = self.config.tree.root_lockfile() {
+ // TODO(nayeemrmn): Iterate every lockfile here for multi-deno.json.
+ if let Some(lockfile) = self
+ .config
+ .tree
+ .root_data()
+ .and_then(|d| d.lockfile.as_ref())
+ {
+ let reqs = npm_reqs_by_scope
+ .entry(self.config.tree.root_scope().cloned())
+ .or_default();
  let lockfile = lockfile.lock();
  for key in lockfile.content.packages.specifiers.keys() {
  if let Some(key) = key.strip_prefix("npm:") {
  if let Ok(req) = PackageReq::from_str(key) {
- npm_reqs.insert(req);
+ reqs.insert(req);
  }
  }
  }
@@ -1372,17 +1389,15 @@ impl Documents {
  // Ensure a @types/node package exists when any module uses a node: specifier.
  // Unlike on the command line, here we just add @types/node to the npm package
  // requirements since this won't end up in the lockfile.
- self.has_injected_types_node_package = has_node_builtin_specifier
- && !npm_reqs.iter().any(|r| r.name == "@types/node");
- if self.has_injected_types_node_package {
- npm_reqs.insert(PackageReq::from_str("@types/node").unwrap());
+ for scope in scopes_with_node_builtin_specifier {
+ let reqs = npm_reqs_by_scope.entry(scope).or_default();
+ if !reqs.iter().any(|r| r.name == "@types/node") {
+ self.has_injected_types_node_package = true;
+ reqs.insert(PackageReq::from_str("@types/node").unwrap());
+ }
  }
 
- self.npm_specifier_reqs = Arc::new({
- let mut reqs = npm_reqs.into_iter().collect::<Vec<_>>();
- reqs.sort();
- reqs
- });
+ self.npm_reqs_by_scope = Arc::new(npm_reqs_by_scope);
  self.dirty = false;
  }
 

cli/lsp/language_server.rs

@@ -806,7 +806,7 @@ impl Inner {
  log::debug!("Skipped workspace walk due to client incapability.");
  return (Default::default(), false);
  }
- let mut workspace_files = Default::default();
+ let mut workspace_files = BTreeSet::default();
  let entry_limit = 1000;
  let mut pending = VecDeque::new();
  let mut entry_count = 0;
@@ -816,10 +816,30 @@ impl Inner {
  .filter_map(|p| specifier_to_file_path(&p.0).ok())
  .collect::<Vec<_>>();
  roots.sort();
- for i in 0..roots.len() {
- if i == 0 || !roots[i].starts_with(&roots[i - 1]) {
- if let Ok(read_dir) = std::fs::read_dir(&roots[i]) {
- pending.push_back((roots[i].clone(), read_dir));
+ let roots = roots
+ .iter()
+ .enumerate()
+ .filter(|(i, root)| *i == 0 || !root.starts_with(&roots[i - 1]))
+ .map(|(_, r)| r.clone())
+ .collect::<Vec<_>>();
+ let mut root_ancestors = BTreeSet::new();
+ for root in roots {
+ for ancestor in root.ancestors().skip(1) {
+ if root_ancestors.insert(ancestor.to_path_buf()) {
+ break;
+ }
+ }
+ if let Ok(read_dir) = std::fs::read_dir(&root) {
+ pending.push_back((root, read_dir));
+ }
+ }
+ for root_ancestor in root_ancestors {
+ for deno_json in ["deno.json", "deno.jsonc"] {
+ let path = root_ancestor.join(deno_json);
+ if path.exists() {
+ if let Ok(specifier) = ModuleSpecifier::from_file_path(path) {
+ workspace_files.insert(specifier);
+ }
  }
  }
  }
@@ -836,17 +856,15 @@ impl Inner {
  let Ok(specifier) = ModuleSpecifier::from_file_path(&path) else {
  continue;
  };
- // TODO(nayeemrmn): Don't walk folders that are `None` here and aren't
- // in a `deno.json` scope.
- if config.settings.specifier_enabled(&specifier) == Some(false) {
- continue;
- }
  let Ok(file_type) = entry.file_type() else {
  continue;
  };
  let Some(file_name) = path.file_name() else {
  continue;
  };
+ if config.settings.specifier_enabled(&specifier) == Some(false) {
+ continue;
+ }
  if file_type.is_dir() {
  let dir_name = file_name.to_string_lossy().to_lowercase();
  // We ignore these directories by default because there is a
@@ -1107,11 +1125,11 @@ impl Inner {
  }
 
  async fn refresh_npm_specifiers(&mut self) {
- let package_reqs = self.documents.npm_package_reqs();
+ let package_reqs = self.documents.npm_reqs_by_scope();
  let resolver = self.resolver.clone();
  // spawn due to the lsp's `Send` requirement
  let handle =
- spawn(async move { resolver.set_npm_package_reqs(&package_reqs).await });
+ spawn(async move { resolver.set_npm_reqs(&package_reqs).await });
  if let Err(err) = handle.await.unwrap() {
  lsp_warn!("Could not set npm package requirements. {:#}", err);
  }
@@ -3428,7 +3446,10 @@ impl Inner {
  roots.extend(
  self
  .documents
- .npm_package_reqs()
+ .npm_reqs_by_scope()
+ .values()
+ .flatten()
+ .collect::<BTreeSet<_>>()
  .iter()
  .map(|req| ModuleSpecifier::parse(&format!("npm:{}", req)).unwrap()),
  );
@@ -3714,21 +3735,29 @@ mod tests {
 
  temp_dir.create_dir_all("root2/folder");
  temp_dir.create_dir_all("root2/sub_folder");
+ temp_dir.create_dir_all("root2/root2.1");
  temp_dir.write("root2/file1.ts", ""); // yes, enabled
  temp_dir.write("root2/file2.ts", ""); // no, not enabled
  temp_dir.write("root2/folder/main.ts", ""); // yes, enabled
  temp_dir.write("root2/folder/other.ts", ""); // no, disabled
  temp_dir.write("root2/sub_folder/a.js", ""); // no, not enabled
  temp_dir.write("root2/sub_folder/b.ts", ""); // no, not enabled
  temp_dir.write("root2/sub_folder/c.js", ""); // no, not enabled
+ temp_dir.write("root2/root2.1/main.ts", ""); // yes, enabled as separate root
 
  temp_dir.create_dir_all("root3/");
  temp_dir.write("root3/mod.ts", ""); // no, not enabled
 
+ temp_dir.create_dir_all("root4_parent/root4");
+ temp_dir.write("root4_parent/deno.json", ""); // yes, enabled as deno.json above root
+ temp_dir.write("root4_parent/root4/main.ts", ""); // yes, enabled
+
  let mut config = Config::new_with_roots(vec![
  temp_dir.uri().join("root1/").unwrap(),
  temp_dir.uri().join("root2/").unwrap(),
+ temp_dir.uri().join("root2/root2.1/").unwrap(),
  temp_dir.uri().join("root3/").unwrap(),
+ temp_dir.uri().join("root4_parent/root4/").unwrap(),
  ]);
  config.set_client_capabilities(ClientCapabilities {
  workspace: Some(Default::default()),
@@ -3756,13 +3785,27 @@ mod tests {
  ..Default::default()
  },
  ),
+ (
+ temp_dir.uri().join("root2/root2.1/").unwrap(),
+ WorkspaceSettings {
+ enable: Some(true),
+ ..Default::default()
+ },
+ ),
  (
  temp_dir.uri().join("root3/").unwrap(),
  WorkspaceSettings {
  enable: Some(false),
  ..Default::default()
  },
  ),
+ (
+ temp_dir.uri().join("root4_parent/root4/").unwrap(),
+ WorkspaceSettings {
+ enable: Some(true),
+ ..Default::default()
+ },
+ ),
  ],
  );
 
@@ -3784,6 +3827,9 @@ mod tests {
  temp_dir.uri().join("root1/sub_dir/mod.ts").unwrap(),
  temp_dir.uri().join("root2/file1.ts").unwrap(),
  temp_dir.uri().join("root2/folder/main.ts").unwrap(),
+ temp_dir.uri().join("root2/root2.1/main.ts").unwrap(),
+ temp_dir.uri().join("root4_parent/deno.json").unwrap(),
+ temp_dir.uri().join("root4_parent/root4/main.ts").unwrap(),
  ])
  );
  }

cli/lsp/resolver.rs

@@ -43,6 +43,8 @@ use deno_semver::package::PackageReq;
 use indexmap::IndexMap;
 use package_json::PackageJsonDepsProvider;
 use std::borrow::Cow;
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::rc::Rc;
@@ -161,13 +163,20 @@ impl LspResolver {
  self.jsr_resolver.as_ref().inspect(|r| r.did_cache());
  }
 
- pub async fn set_npm_package_reqs(
+ pub async fn set_npm_reqs(
  &self,
- reqs: &[PackageReq],
+ reqs: &BTreeMap<Option<ModuleSpecifier>, BTreeSet<PackageReq>>,
  ) -> Result<(), AnyError> {
+ let reqs = reqs
+ .values()
+ .flatten()
+ .collect::<BTreeSet<_>>()
+ .into_iter()
+ .cloned()
+ .collect::<Vec<_>>();
  if let Some(npm_resolver) = self.npm_resolver.as_ref() {
  if let Some(npm_resolver) = npm_resolver.as_managed() {
- return npm_resolver.set_package_reqs(reqs).await;
+ return npm_resolver.set_package_reqs(&reqs).await;
  }
  }
  Ok(())