feat: Support multiple repMap combinations (#4270)

integration-tests/snapshots/MartinThoma/LaTeX-examples/report.yaml

@@ -5,7 +5,7 @@ Args: ["**/*.{md,tex}"]
 Summary:
  files: 1407
  filesWithIssues: 484
- issues: 4887
+ issues: 4877
  errors: 0
 Errors: []
 
@@ -143,7 +143,6 @@ issues:
  - "documents/Analysis%20III/Kapitel-1.tex:146:23 Borelsche U \\textbf{Borelsche $\\sigma$-Algebra} auf"
  - "documents/Analysis%20III/Kapitel-1.tex:148:54 Borel U Mengen} oder \\textbf{Borel-Mengen}."
  - "documents/Analysis%20III/Kapitel-1.tex:476:73 Subadditivität U {j})}\\) (\\(\\sigma\\)-Subadditivität)"
- - "documents/Analysis%20III/Kapitel-1.tex:484:14 muesste U % Eigentlich muesste es in folgender Zeile"
  - "documents/Analysis%20III/Kapitel-10.tex:164:21 Integrationen U Die Reihenfolge der Integrationen darf beliebig vertauscht"
  - "documents/Analysis%20III/Kapitel-11.tex:8:9 Diffeomorphismus U \\textbf{Diffeomorphismus} genau dann wenn \\("
  - "documents/Analysis%20III/Kapitel-12.tex:41:150 Tangentialvektor U )\\in\\mdr^n$ \\textbf{Tangentialvektor} von $\\gamma$ in $t"
@@ -696,7 +695,6 @@ issues:
  - "documents/kit-agb/kit-agb.tex:43:16 Vertiefungsfach U $\\boxtimes$ Vertiefungsfach \\\\"
  - "documents/kit-agb/kit-agb.tex:45:19 Vertiefungs U %% Namen des Wahl/Vertiefungs/Ergaenzungsfachs hier"
  - "documents/kit-agb/kit-agb.tex:61:9 gehts U %% Hier gehts weiter:"
- - "documents/kit-agb/kit-agb.tex:91:10 Pruefungsdauer U %% Bitte Pruefungsdauer eintragen"
  - "documents/kit-agb/kit-agb.tex:102:18 nachgehackt U immer wieder nachgehackt."
  - "documents/kit-agb/kit-agb.tex:124:56 Komilitonen U 2 Treffen à 5h mit Komilitonen."
  - "documents/kit-agb/kit-agb.tex:177:72 Metacost U Daten. Es wird z.b. von Metacost"

integration-tests/snapshots/MartinThoma/LaTeX-examples/snapshot.txt

@@ -3,7 +3,7 @@ Repository: MartinThoma/LaTeX-examples
 Url: "https://github.com/MartinThoma/LaTeX-examples.git"
 Args: ["**/*.{md,tex}"]
 Lines:
- CSpell: Files checked: 1407, Issues found: 4887 in 484 files
+ CSpell: Files checked: 1407, Issues found: 4877 in 484 files
  exit code: 1
 ./README.md:12:35 - Unknown word (Wikpedia) -- can be found on [my Wikpedia Commons user page](http
 ./README.md:20:26 - Unknown word (seperate) -- Every LaTeX file is in a seperate folder and has its own
@@ -138,7 +138,6 @@ Lines:
 ./documents/Analysis III/Kapitel-1.tex:146:23 - Unknown word (Borelsche) -- \textbf{Borelsche $\sigma$-Algebra} auf
 ./documents/Analysis III/Kapitel-1.tex:148:54 - Unknown word (Borel) -- Mengen} oder \textbf{Borel-Mengen}.
 ./documents/Analysis III/Kapitel-1.tex:476:73 - Unknown word (Subadditivität) -- {j})}\) (\(\sigma\)-Subadditivität)
-./documents/Analysis III/Kapitel-1.tex:484:14 - Unknown word (muesste) -- % Eigentlich muesste es in folgender Zeile
 ./documents/Analysis III/Kapitel-10.tex:164:21 - Unknown word (Integrationen) -- Die Reihenfolge der Integrationen darf beliebig vertauscht
 ./documents/Analysis III/Kapitel-11.tex:8:9 - Unknown word (Diffeomorphismus) -- \textbf{Diffeomorphismus} genau dann wenn \(
 ./documents/Analysis III/Kapitel-12.tex:41:150 - Unknown word (Tangentialvektor) -- )\in\mdr^n$ \textbf{Tangentialvektor} von $\gamma$ in $t
@@ -908,7 +907,6 @@ Lines:
 ./documents/kit-agb/kit-agb.tex:43:16 - Unknown word (Vertiefungsfach) -- $\boxtimes$ Vertiefungsfach \\
 ./documents/kit-agb/kit-agb.tex:45:19 - Unknown word (Vertiefungs) -- %% Namen des Wahl/Vertiefungs/Ergaenzungsfachs hier
 ./documents/kit-agb/kit-agb.tex:61:9 - Unknown word (gehts) -- %% Hier gehts weiter:
-./documents/kit-agb/kit-agb.tex:91:10 - Unknown word (Pruefungsdauer) -- %% Bitte Pruefungsdauer eintragen
 ./documents/kit-mathe-template/README.md:8:44 - Unknown word (Kühnleins) -- template, sign it, add Dr. Kühnleins permission and give
 ./documents/kit-mathe-template/README.md:9:23 - Unknown word (Gheta) -- to ... hm ... [Dr. Gheta](http:https://www.informatik
 ./documents/kit-mathe-template/kit-mathe-template.tex:53:24 - Unknown word (MATHBAST) -- \item \texttt{[MATHBAST01]} Einführung in die

packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionary.ts

@@ -155,6 +155,13 @@ export interface SpellingDictionary extends DictionaryInfo {
  suggest(word: string, suggestOptions: SuggestOptions): SuggestionResult[];
  genSuggestions(collector: SuggestionCollector, suggestOptions: SuggestOptions): void;
  mapWord(word: string): string;
+ /**
+ * Generates all possible word combinations by applying `repMap`.
+ * This acts a bit like brace expansions in globs.
+ * @param word - the word to map
+ * @returns array of adjusted words.
+ */
+ remapWord?: (word: string) => string[];
  readonly size: number;
  readonly isDictionaryCaseSensitive: boolean;
  getErrors?(): Error[];

packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts

@@ -1,19 +1,28 @@
-import { createMapper } from '../util/repMap';
-import { __testing__ } from './SpellingDictionaryFromTrie';
+import { buildTrieFast } from 'cspell-trie-lib';
+
+import { __testing__, SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie';
 
 const { outerWordForms } = __testing__;
 
 // cspell:ignore guenstig günstig Bundesstaat Bundeßtaat
+// cspell:ignore Goerresstraße, Goerreßtraße, Görresstraße, Görreßtraße
 
 describe('SpellingDictionaryFromTrie', () => {
  test.each`
- word | repMap | expected
- ${'hello'} | ${undefined} | ${['hello']}
- ${'guenstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['guenstig', 'günstig']}
- ${'günstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['günstig', 'günstig'.normalize('NFD')]}
- ${'Bundesstaat'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['Bundesstaat', 'Bundeßtaat']}
+ word | repMap | expected
+ ${'hello'} | ${undefined} | ${['hello']}
+ ${'guenstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['guenstig', 'günstig']}
+ ${'günstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['günstig', N('günstig')]}
+ ${'Bundesstaat'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['Bundesstaat', 'Bundeßtaat']}
+ ${'Goerresstraße'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['Goerresstraße', 'Goerreßtraße', 'Görresstraße', 'Görreßtraße']}
  `('outerWordForms $word', ({ word, repMap, expected }) => {
- const mapWord = createMapper(repMap);
- expect(outerWordForms(word, mapWord ?? ((a) => a))).toEqual(new Set(expected));
+ const trie = buildTrieFast([]);
+ const dict = new SpellingDictionaryFromTrie(trie, 'test', { repMap });
+ const mapWord = dict.remapWord || ((a) => [dict.mapWord(a)]);
+ expect(outerWordForms(word, mapWord)).toEqual(new Set(expected));
  });
 });
+
+function N(s: string, mode: 'NFD' | 'NFC' = 'NFD') {
+ return s.normalize(mode);
+}

packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.ts

@@ -10,7 +10,7 @@ import { CompoundWordsMethod, importTrie, suggestionCollector, Trie } from 'cspe
 
 import { autoCache, createCache01 } from '../util/AutoCache';
 import { clean } from '../util/clean';
-import { createMapper } from '../util/repMap';
+import { createMapper, createRepMapper } from '../util/repMap';
 import * as Defaults from './defaults';
 import type {
  FindResult,
@@ -39,6 +39,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
  readonly knownWords = new Set<string>();
  readonly unknownWords = new Set<string>();
  readonly mapWord: (word: string) => string;
+ readonly remapWord: (word: string) => string[];
  readonly type = 'SpellingDictionaryFromTrie';
  readonly isDictionaryCaseSensitive: boolean;
  readonly containsNoSuggestWords: boolean;
@@ -53,6 +54,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
  size?: number
  ) {
  this.mapWord = createMapper(options.repMap, options.dictionaryInformation?.ignore);
+ this.remapWord = createRepMapper(options.repMap, options.dictionaryInformation?.ignore);
  this.isDictionaryCaseSensitive = options.caseSensitive ?? !trie.isLegacy;
  this.containsNoSuggestWords = options.noSuggest || false;
  this._size = size || 0;
@@ -109,7 +111,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
  useCompounds: number | boolean | undefined,
  ignoreCase: boolean
  ): FindAnyFormResult | undefined {
- const outerForms = outerWordForms(word, this.mapWord);
+ const outerForms = outerWordForms(word, this.remapWord ? this.remapWord : (word) => [this.mapWord(word)]);
 
  for (const form of outerForms) {
  const r = this._findAnyForm(form, useCompounds, ignoreCase);
@@ -265,11 +267,11 @@ function findCache(fn: FindFunction, size = 2000): FindFunction {
  return find;
 }
 
-function outerWordForms(word: string, mapWord: (word: string) => string): Set<string> {
+function outerWordForms(word: string, mapWord: (word: string) => string[]): Set<string> {
  const forms = pipe(
  [word],
  opConcatMap((word) => [word, word.normalize('NFC'), word.normalize('NFD')]),
- opConcatMap((word) => [word, mapWord(word)])
+ opConcatMap((word) => [word, ...mapWord(word)])
  );
 
  return new Set(forms);

packages/cspell-dictionary/src/util/repMap.test.ts

@@ -1,6 +1,6 @@
 import { __testing__, createMapper } from './repMap';
 
-const { createMapperRegExp, charsetToRepMap } = __testing__;
+const { createMapperRegExp, charsetToRepMap, createTrie, calcAllEdits, applyEdits } = __testing__;
 
 describe('ReMap Tests', () => {
  test('empty replace map', () => {
@@ -109,3 +109,51 @@ describe('ReMap Tests', () => {
  expect(reg).toEqual(expected);
  });
 });
+
+describe('RepMapper', () => {
+ test.each`
+ repMap | ignoreChars | expected
+ ${undefined} | ${undefined} | ${{}}
+ ${[['a', 'b']]} | ${undefined} | ${{ children: { a: { rep: ['b'] } } }}
+ ${[['a', 'b'], ['a', 'b']]} | ${undefined} | ${{ children: { a: { rep: ['b'] } } }}
+ ${[['a', 'b'], ['a', 'c']]} | ${undefined} | ${{ children: { a: { rep: ['b', 'c'] } } }}
+ ${[['a', 'b'], ['a', 'c']]} | ${'a'} | ${{ children: { a: { rep: ['b', 'c', ''] } } }}
+ ${[['a', 'b'], ['a', 'c']]} | ${'i'} | ${{ children: { a: { rep: ['b', 'c'] }, i: { rep: [''] } } }}
+ ${[['a', 'b'], ['a', 'c']]} | ${'i'} | ${{ children: { a: { rep: ['b', 'c'] }, i: { rep: [''] } } }}
+ ${[['a|i', 'b'], ['a', 'c']]} | ${'i'} | ${{ children: { a: { rep: ['b', 'c'] }, i: { rep: ['b', ''] } } }}
+ `('createTrie', ({ repMap, ignoreChars, expected }) => {
+ expect(createTrie(repMap, ignoreChars)).toEqual(expected);
+ });
+
+ test.each`
+ repMap | ignoreChars | word | expected
+ ${undefined} | ${undefined} | ${'hello'} | ${[]}
+ ${[['e', 'é']]} | ${undefined} | ${'hello'} | ${[{ b: 1, e: 2, r: 'é' }]}
+ ${[['e', 'é'], ['o', 'ó']]} | ${undefined} | ${'hello'} | ${[{ b: 1, e: 2, r: 'é' }, { b: 4, e: 5, r: 'ó' }]}
+ ${[['ll', 'y'], ['ll', 'el']]} | ${undefined} | ${'hello'} | ${[{ b: 2, e: 4, r: 'y' }, { b: 2, e: 4, r: 'el' }]}
+ ${[['f', 'ph'], ['ph', 'f']]} | ${undefined} | ${'phone'} | ${[{ b: 0, e: 2, r: 'f' }]}
+ `('calcAllEdits', ({ repMap, ignoreChars, word, expected }) => {
+ const root = createTrie(repMap, ignoreChars);
+ expect(calcAllEdits(root, word)).toEqual(expected);
+ });
+
+ // cspell:ignore héllo helló hélló heyo heelo fone phoné phöne
+ test.each`
+ repMap | ignoreChars | word | expected
+ ${undefined} | ${undefined} | ${'hello'} | ${['hello']}
+ ${[['e', 'é']]} | ${undefined} | ${'hello'} | ${['hello', 'héllo']}
+ ${[['e', 'é'], ['o', 'ó']]} | ${undefined} | ${'hello'} | ${['hello', 'helló', 'héllo', 'hélló']}
+ ${[['ll', 'y'], ['ll', 'el']]} | ${undefined} | ${'hello'} | ${['hello', 'heyo', 'heelo']}
+ ${[['f', 'ph'], ['ph', 'f']]} | ${undefined} | ${'phone'} | ${['phone', 'fone']}
+ ${[]} | ${'\u0300-\u0308'} | ${N('phoné')} | ${[N('phoné'), 'phone']}
+ ${[]} | ${'\u0300-\u0308'} | ${N('phöne')} | ${[N('phöne'), 'phone']}
+ `('applyEdits', ({ repMap, ignoreChars, word, expected }) => {
+ const root = createTrie(repMap, ignoreChars);
+ const edits = calcAllEdits(root, word);
+ expect(applyEdits(word, edits)).toEqual(expected);
+ });
+});
+
+function N(s: string, mode: 'NFD' | 'NFC' = 'NFD') {
+ return s.normalize(mode);
+}

packages/cspell-dictionary/src/util/repMap.ts

@@ -1,13 +1,15 @@
-import type { CharacterSet, ReplaceMap } from '@cspell/cspell-types';
+import type { CharacterSet, ReplaceEntry, ReplaceMap } from '@cspell/cspell-types';
+import { expandCharacterSet } from 'cspell-trie-lib';
 
 import { escapeRegEx } from './regexHelper';
+import { isDefined } from './util';
 
 export type ReplaceMapper = (src: string) => string;
 
 export function createMapper(repMap: ReplaceMap | undefined, ignoreCharset?: string): ReplaceMapper {
  if (!repMap && !ignoreCharset) return (a) => a;
  repMap = repMap || [];
- const charsetMap = charsetToRepMap(ignoreCharset);
+ const charsetMap = charsetToRepMapRegEx(ignoreCharset);
  if (charsetMap) {
  repMap = repMap.concat(charsetMap);
  }
@@ -30,14 +32,29 @@ export function createMapper(repMap: ReplaceMap | undefined, ignoreCharset?: str
  };
 }
 
-function charsetToRepMap(charset: CharacterSet | undefined, replaceWith = ''): ReplaceMap | undefined {
+function charsetToRepMapRegEx(charset: CharacterSet | undefined, replaceWith = ''): ReplaceMap | undefined {
  if (!charset) return undefined;
 
  return charset
  .split('|')
  .map((chars) => `[${chars.replace(/[\][\\]/g, '\\$&')}]`)
  .map((map) => [map, replaceWith]);
 }
+function charsetToRepMap(charset: undefined, replaceWith?: string): undefined;
+function charsetToRepMap(charset: CharacterSet, replaceWith?: string): ReplaceMap;
+function charsetToRepMap(charset: CharacterSet | undefined, replaceWith?: string): ReplaceMap | undefined;
+function charsetToRepMap(charset: CharacterSet | undefined, replaceWith = ''): ReplaceMap | undefined {
+ if (!charset) return undefined;
+
+ return charset
+ .split('|')
+ .flatMap((chars) => [...expandCharacterSet(chars)])
+ .map((char) => [char, replaceWith]);
+}
+
+function expandReplaceMap(repMap: ReplaceMap): ReplaceMap {
+ return repMap.flatMap(([from, replaceWith]) => from.split('|').map((w) => [w, replaceWith] as ReplaceEntry));
+}
 
 function createMapperRegExp(repMap: ReplaceMap): RegExp {
  const filteredMap = repMap.filter(([match, _]) => !!match);
@@ -66,7 +83,110 @@ function createMapperRegExp(repMap: ReplaceMap): RegExp {
  return regEx;
 }
 
+interface RepTrieNode {
+ rep?: string[];
+ children?: Record<string, RepTrieNode>;
+}
+
+interface Edit {
+ b: number;
+ e: number;
+ r: string;
+}
+
+export function createRepMapper(repMap: ReplaceMap | undefined, ignoreCharset?: string): (word: string) => string[] {
+ if (!repMap && !ignoreCharset) return (word) => [word];
+
+ const trie = createTrie(repMap, ignoreCharset);
+
+ // const root = createTrie(repMap, ignoreCharset);
+ return (word) => {
+ const edits = calcAllEdits(trie, word);
+ return applyEdits(word, edits);
+ };
+}
+
+function applyEdits(word: string, edits: Edit[]): string[] {
+ if (!edits.length) return [word];
+
+ // Prepare
+ const letterEdits: { edits: Edit[]; suffixes: string[] }[] = [];
+ for (let i = 0; i < word.length; ++i) {
+ letterEdits[i] = { edits: [{ b: i, e: i + 1, r: word[i] }], suffixes: [] };
+ }
+ letterEdits[word.length] = { edits: [], suffixes: [''] };
+
+ // Add edits
+ for (const edit of edits) {
+ const le = letterEdits[edit.b];
+ le.edits.push(edit);
+ }
+
+ // Apply edits in reverse
+ for (let i = word.length - 1; i >= 0; --i) {
+ const le = letterEdits[i];
+ const sfx = le.suffixes;
+ for (const edit of le.edits) {
+ const pfx = edit.r;
+ const nSfx = letterEdits[edit.e].suffixes;
+ for (const s of nSfx) {
+ sfx.push(pfx + s);
+ }
+ }
+ }
+
+ const results = new Set(letterEdits[0].suffixes);
+
+ return [...results];
+}
+
+function calcAllEdits(root: RepTrieNode, word: string): Edit[] {
+ const edits: Edit[] = [];
+
+ function walk(node: RepTrieNode, b: number, e: number) {
+ if (node.rep) {
+ node.rep.forEach((r) => edits.push({ b, e, r }));
+ }
+ if (e === word.length || !node.children) return;
+ const n = node.children[word[e]];
+ if (!n) return;
+ walk(n, b, e + 1);
+ }
+
+ for (let i = 0; i < word.length; ++i) {
+ walk(root, i, i);
+ }
+
+ return edits;
+}
+
+function createTrie(repMap: ReplaceMap | undefined, ignoreCharset?: string): RepTrieNode {
+ const combined = [repMap, charsetToRepMap(ignoreCharset)].filter(isDefined).flatMap((a) => a);
+ const expanded = expandReplaceMap(combined);
+
+ const trieRoot: RepTrieNode = Object.create(null);
+
+ expanded.forEach(([match, replaceWith]) => addToTrie(trieRoot, match, replaceWith));
+ return trieRoot;
+}
+
+function addToTrie(node: RepTrieNode, match: string, replaceWith: string) {
+ while (match) {
+ const children: Record<string, RepTrieNode> = node.children || (node.children = Object.create(null));
+ const k = match[0];
+ const childNode = children[k] || (children[k] = Object.create(null));
+ node = childNode;
+ match = match.slice(1);
+ }
+ const s = new Set(node.rep || []);
+ s.add(replaceWith);
+ node.rep = [...s];
+}
+
 export const __testing__ = {
- charsetToRepMap,
+ charsetToRepMap: charsetToRepMapRegEx,
  createMapperRegExp,
+ createTrie,
+ calcAllEdits,
+ applyEdits,
 };

packages/cspell-trie-lib/src/__snapshots__/index.test.ts.snap

@@ -32,6 +32,7 @@ exports[`Validate index.ts > Track changes to the API. 1`] = `
  "defaultTrieOptions",
  "editDistance",
  "editDistanceWeighted",
+ "expandCharacterSet",
  "findNode",
  "has",
  "hintedWalker",

packages/cspell-trie-lib/src/lib/index.ts

@@ -47,5 +47,6 @@ export { isDefined } from './utils/isDefined.js';
 export { mergeDefaults } from './utils/mergeDefaults.js';
 export { mergeOptionalWithDefaults } from './utils/mergeOptionalWithDefaults.js';
 export { normalizeWord, normalizeWordForCaseInsensitive, normalizeWordToLowercase } from './utils/normalizeWord.js';
+export { expandCharacterSet } from './utils/text.js';
 export type { HintedWalkerIterator, Hinting, WalkerIterator, YieldResult } from './walker/index.js';
 export { CompoundWordsMethod, hintedWalker, JOIN_SEPARATOR, walker, WORD_SEPARATOR } from './walker/index.js';