fix: be able to report on glob source. (#1001)

* fix: be able to report on glob source.

- Add an optional source field to the glob definition.
- Handle circular imports.

* Update cspell.config.json

packages/cspell-lib/cspell.config.json

@@ -1,16 +1,14 @@
 {
+ "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/master/cspell.schema.json",
  "version": "0.2",
  "id": "cspell-package-config",
  "name": "cspell Package Config",
  "language": "en",
  "words": ["gensequence"],
  "maxNumberOfProblems": 100,
  "ignorePaths": [
- "dictionaries/**",
- "migrated_dictionaries/**",
  "node_modules/**",
  "coverage/**",
- ".git/**",
  "dist/**",
  "package.json",
  "**/package.json",

packages/cspell-lib/samples/linked/cspell.circularA.json

@@ -0,0 +1,14 @@
+{
+ "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/master/cspell.schema.json",
+ "name": "circular file A",
+ "version": "0.2",
+ "ignorePaths": [
+ "node_modules"
+ ],
+ "words": [
+ "aa"
+ ],
+ "import": [
+ "./cspell.circularB.json"
+ ]
+}

packages/cspell-lib/samples/linked/cspell.circularB.json

@@ -0,0 +1,14 @@
+{
+ "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/master/cspell.schema.json",
+ "name": "circular file B",
+ "version": "0.2",
+ "ignorePaths": [
+ "node_modules"
+ ],
+ "words": [
+ "bb"
+ ],
+ "import": [
+ "./cspell.circularA.json"
+ ]
+}

packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts

@@ -21,7 +21,8 @@ import * as path from 'path';
 
 const { normalizeSettings } = __testing__;
 
-const samplesDir = path.resolve(path.join(__dirname, '../../samples'));
+const rootCspellLib = path.resolve(path.join(__dirname, '../..'));
+const samplesDir = path.resolve(rootCspellLib, 'samples');
 const samplesSrc = path.join(samplesDir, 'src');
 
 jest.mock('../util/logger');
@@ -239,6 +240,44 @@ describe('Validate CSpellSettingsServer', () => {
  const sourceNames = sources.map((s) => s.name || '?');
  expect(sourceNames).toEqual(expect.arrayContaining([_defaultSettings.name]));
  });
+
+ test('loading circular imports (readSettings)', async () => {
+ const configFile = path.join(samplesDir, 'linked/cspell.circularA.json');
+ const config = readSettings(configFile);
+ expect(config?.ignorePaths).toEqual(
+ expect.arrayContaining([
+ {
+ glob: 'node_modules',
+ root: path.dirname(configFile),
+ source: configFile,
+ },
+ ])
+ );
+ const errors = extractImportErrors(config);
+ expect(errors).toEqual([]);
+
+ const sources = getSources(config);
+ expect(sources.length).toBe(2);
+ });
+
+ test('loading circular imports (loadConfig)', async () => {
+ const configFile = path.join(samplesDir, 'linked/cspell.circularA.json');
+ const config = await loadConfig(configFile);
+ expect(config?.ignorePaths).toEqual(
+ expect.arrayContaining([
+ {
+ glob: 'node_modules',
+ root: path.dirname(configFile),
+ source: configFile,
+ },
+ ])
+ );
+ const errors = extractImportErrors(config);
+ expect(errors).toEqual([]);
+
+ const sources = getSources(config);
+ expect(sources.length).toBe(2);
+ });
 });
 
 describe('Validate Overrides', () => {
@@ -283,10 +322,14 @@ describe('Validate Glob resolution', () => {
  expect(sampleSettings.globRoot).toBe(__dirname);
  expect(sampleSettingsV1.globRoot).toBe(process.cwd());
  expect(sampleSettings.ignorePaths).toEqual(
- expect.arrayContaining([{ glob: 'node_modules', root: sampleSettings.globRoot }])
+ expect.arrayContaining([
+ { glob: 'node_modules', root: sampleSettings.globRoot, source: sampleSettingsFilename },
+ ])
  );
  expect(sampleSettingsV1.ignorePaths).toEqual(
- expect.arrayContaining([{ glob: 'node_modules', root: sampleSettingsV1.globRoot }])
+ expect.arrayContaining([
+ { glob: 'node_modules', root: sampleSettingsV1.globRoot, source: sampleSettingsFilename },
+ ])
  );
  });
 
@@ -313,6 +356,33 @@ describe('Validate Glob resolution', () => {
  delete settingsV1.version;
  expect(settingsV1).toEqual(settingsV);
  });
+
+ test('globs from config file (search)', async () => {
+ const config = await searchForConfig(__dirname);
+ expect(config?.ignorePaths).toEqual(
+ expect.arrayContaining([
+ {
+ glob: 'node_modules/**',
+ root: rootCspellLib,
+ source: path.join(rootCspellLib, 'cspell.config.json'),
+ },
+ ])
+ );
+ });
+
+ test('globs from config file (readSettings)', async () => {
+ const configFile = path.join(rootCspellLib, 'cspell.config.json');
+ const config = readSettings(configFile);
+ expect(config?.ignorePaths).toEqual(
+ expect.arrayContaining([
+ {
+ glob: 'node_modules/**',
+ root: rootCspellLib,
+ source: configFile,
+ },
+ ])
+ );
+ });
 });
 
 describe('Validate search/load config files', () => {
@@ -445,7 +515,7 @@ const rawSampleSettings: CSpellUserSettings = {
 };
 
 const rawSampleSettingsV1: CSpellUserSettings = { ...rawSampleSettings, version: '0.1' };
+const sampleSettingsFilename = __filename;
+const sampleSettings: CSpellUserSettings = normalizeSettings(rawSampleSettings, sampleSettingsFilename);
 
-const sampleSettings: CSpellUserSettings = normalizeSettings(rawSampleSettings, __filename);
-
-const sampleSettingsV1: CSpellUserSettings = normalizeSettings(rawSampleSettingsV1, __filename);
+const sampleSettingsV1: CSpellUserSettings = normalizeSettings(rawSampleSettingsV1, sampleSettingsFilename);

packages/cspell-lib/src/Settings/CSpellSettingsServer.ts

@@ -18,7 +18,7 @@ import { cosmiconfig, cosmiconfigSync, OptionsSync as CosmicOptionsSync, Options
 import { GlobMatcher } from 'cspell-glob';
 import { ImportError } from './ImportError';
 
-const currentSettingsFileVersion = '0.1';
+const currentSettingsFileVersion = '0.2';
 
 export const sectionCSpell = 'cSpell';
 
@@ -104,9 +104,16 @@ function readConfig(fileRef: ImportFileRef): CSpellSettings {
  * @param pathToSettingsFile - path to the source file of the configuration settings.
  */
 function normalizeSettings(rawSettings: CSpellSettings, pathToSettingsFile: string): CSpellSettings {
+ const id =
+ rawSettings.id ||
+ [path.basename(path.dirname(pathToSettingsFile)), path.basename(pathToSettingsFile)].join('/');
+ const name = rawSettings.name || id;
+
  // Fix up dictionaryDefinitions
  const settings = {
  ...rawSettings,
+ id,
+ name,
  globRoot: resolveGlobRoot(rawSettings, pathToSettingsFile),
  languageSettings: normalizeLanguageSettings(rawSettings.languageSettings),
  };
@@ -117,7 +124,7 @@ function normalizeSettings(rawSettings: CSpellSettings, pathToSettingsFile: stri
 
  const imports = typeof settings.import === 'string' ? [settings.import] : settings.import || [];
  const source: Source = settings.source || {
- name: settings.name || settings.id || pathToSettingsFile,
+ name: settings.name,
  filename: pathToSettingsFile,
  };
 
@@ -167,9 +174,10 @@ function importSettings(fileRef: ImportFileRef, defaultValues: CSpellSettings =
  return cached;
  }
  const id = [path.basename(path.dirname(filename)), path.basename(filename)].join('/');
- const finalizeSettings: CSpellSettings = { id, __importRef: importRef };
+ const name = id;
+ const finalizeSettings: CSpellSettings = { id, name, __importRef: importRef };
  cachedFiles.set(filename, finalizeSettings); // add an empty entry to prevent circular references.
- const settings: CSpellSettings = { ...defaultValues, id, ...readConfig(importRef) };
+ const settings: CSpellSettings = { ...defaultValues, id, name, ...readConfig(importRef) };
 
  Object.assign(finalizeSettings, normalizeSettings(settings, filename));
  const finalizeSrc: Source = { name: path.basename(filename), ...finalizeSettings.source };
@@ -218,6 +226,17 @@ async function normalizeSearchForConfigResult(
  }
 
  const filepath = result?.filepath;
+ if (filepath) {
+ const cached = cachedFiles.get(filepath);
+ if (cached) {
+ return {
+ config: cached,
+ filepath,
+ error,
+ };
+ }
+ }
+
  const { config = {} } = result || {};
  const filename = result?.filepath ?? searchPath;
  const importRef: ImportFileRef = { filename: filename, error };
@@ -226,6 +245,7 @@ async function normalizeSearchForConfigResult(
  const name = result?.filepath ? id : `Config not found: ${id}`;
  const finalizeSettings: CSpellSettings = { id, name, __importRef: importRef };
  const settings: CSpellSettings = { id, ...config };
+ cachedFiles.set(filename, finalizeSettings); // add an empty entry to prevent circular references.
  Object.assign(finalizeSettings, normalizeSettings(settings, filename));
 
  return {
@@ -243,6 +263,10 @@ export function searchForConfig(searchFrom?: string): Promise<CSpellSettings | u
 }
 
 export function loadConfig(file: string): Promise<CSpellSettings> {
+ const cached = cachedFiles.get(path.resolve(file));
+ if (cached) {
+ return Promise.resolve(cached);
+ }
  return normalizeSearchForConfigResult(file, cspellConfigExplorer.load(file)).then((r) => r.config);
 }
 
@@ -539,12 +563,22 @@ function mergeSources(left: CSpellSettings, right: CSpellSettings): Source {
  * @param settings the settings to search
  */
 export function getSources(settings: CSpellSettings): CSpellSettings[] {
- if (!settings.source?.sources?.length) {
- return [settings];
+ const visited = new Set<CSpellSettings>();
+ const sources: CSpellSettings[] = [];
+
+ function _walkSourcesTree(settings: CSpellSettings | undefined): void {
+ if (!settings || visited.has(settings)) return;
+ visited.add(settings);
+ if (!settings.source?.sources?.length) {
+ sources.push(settings);
+ return;
+ }
+ settings.source.sources.forEach(_walkSourcesTree);
  }
- const left = settings.source.sources[0];
- const right = settings.source.sources[1];
- return right ? getSources(left).concat(getSources(right)) : getSources(left);
+
+ _walkSourcesTree(settings);
+
+ return sources;
 }
 
 type Imports = CSpellSettings['__imports'];
@@ -589,20 +623,32 @@ function resolveGlobRoot(settings: CSpellSettings, pathToSettingsFile: string):
  return globRoot;
 }
 
-function toGlobDef(g: undefined, root: string | undefined): undefined;
-function toGlobDef(g: Glob, root: string | undefined): GlobDef;
-function toGlobDef(g: Glob[], root: string | undefined): GlobDef[];
-function toGlobDef(g: Glob | Glob[], root: string | undefined): GlobDef | GlobDef[];
-function toGlobDef(g: Glob | Glob[] | undefined, root: string | undefined): GlobDef | GlobDef[] | undefined {
+function toGlobDef(g: undefined, root: string | undefined, source: string | undefined): undefined;
+function toGlobDef(g: Glob, root: string | undefined, source: string | undefined): GlobDef;
+function toGlobDef(g: Glob[], root: string | undefined, source: string | undefined): GlobDef[];
+function toGlobDef(g: Glob | Glob[], root: string | undefined, source: string | undefined): GlobDef | GlobDef[];
+function toGlobDef(
+ g: Glob | Glob[] | undefined,
+ root: string | undefined,
+ source: string | undefined
+): GlobDef | GlobDef[] | undefined {
  if (g === undefined) return undefined;
  if (Array.isArray(g)) {
- return g.map((g) => toGlobDef(g, root));
+ return g.map((g) => toGlobDef(g, root, source));
  }
- if (typeof g === 'string')
- return {
- glob: g,
+ if (typeof g === 'string') {
+ return toGlobDef(
+ {
+ glob: g,
+ root,
+ },
  root,
- };
+ source
+ );
+ }
+ if (source) {
+ return { ...g, source };
+ }
  return g;
 }
 
@@ -638,7 +684,7 @@ interface NormalizeOverridesResult {
 function normalizeOverrides(settings: NormalizeOverrides, pathToSettingsFile: string): NormalizeOverridesResult {
  const { globRoot = path.dirname(pathToSettingsFile) } = settings;
  const overrides = settings.overrides?.map((override) => {
- const filename = toGlobDef(override.filename, globRoot);
+ const filename = toGlobDef(override.filename, globRoot, pathToSettingsFile);
  const { dictionaryDefinitions, languageSettings } = normalizeDictionaryDefs(override, pathToSettingsFile);
  return util.clean({
  ...override,
@@ -678,7 +724,7 @@ function normalizeSettingsGlobs(
  const { globRoot = path.dirname(pathToSettingsFile) } = settings;
  if (settings.ignorePaths === undefined) return {};
 
- const ignorePaths = toGlobDef(settings.ignorePaths, globRoot);
+ const ignorePaths = toGlobDef(settings.ignorePaths, globRoot, pathToSettingsFile);
  return {
  ignorePaths,
  };

packages/cspell-types/src/settings/CSpellSettingsDef.ts

@@ -425,6 +425,9 @@ export interface GlobDef {
 
  /** Optional root to use when matching the glob. Defaults to current working dir. */
  root?: string;
+
+ /** Optional source of the glob, used when merging settings to determine the origin. */
+ source?: string;
 }
 
 /**