hunspell: add Suggester#proceedPastRep to avoid losing relevant sugge…

…stions (#13612)

* hunspell: add Suggester#proceedPastRep to avoid losing relevant suggestions

lucene/CHANGES.txt

@@ -253,6 +253,8 @@ API Changes
 
 * GITHUB#13469: Expose FlatVectorsFormat as a first-class format; can be configured using a custom Codec. (Michael Sokolov) 
 
+* GITHUB#13612: Hunspell: add Suggester#proceedPastRep to avoid losing relevant suggestions. (Peter Gromov)
+
 New Features
 ---------------------
 

lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java

@@ -31,6 +31,7 @@ class ModifyingSuggester {
  private final String misspelled;
  private final WordCase wordCase;
  private final FragmentChecker fragmentChecker;
+ private final boolean proceedPastRep;
  private final char[] tryChars;
  private final Hunspell speller;
 
@@ -39,13 +40,15 @@ class ModifyingSuggester {
  LinkedHashSet<Suggestion> result,
  String misspelled,
  WordCase wordCase,
- FragmentChecker checker) {
+ FragmentChecker checker,
+ boolean proceedPastRep) {
  this.speller = speller;
  tryChars = speller.dictionary.tryChars.toCharArray();
  this.result = result;
  this.misspelled = misspelled;
  this.wordCase = wordCase;
  fragmentChecker = checker;
+ this.proceedPastRep = proceedPastRep;
  }
 
  /**
@@ -125,9 +128,9 @@ private boolean tryVariationsOf(String word) {
  boolean hasGoodSuggestions = trySuggestion(word.toUpperCase(Locale.ROOT));
 
  GradedSuggestions repResult = tryRep(word);
- if (repResult == GradedSuggestions.Best) return true;
+ if (repResult == GradedSuggestions.Best && !proceedPastRep) return true;
 
- hasGoodSuggestions |= repResult == GradedSuggestions.Normal;
+ hasGoodSuggestions |= repResult != GradedSuggestions.None;
 
  if (!speller.dictionary.mapTable.isEmpty()) {
  enumerateMapReplacements(word, "", 0);

lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java

@@ -53,16 +53,21 @@ public class Suggester {
  private final Dictionary dictionary;
  private final SuggestibleEntryCache suggestibleCache;
  private final FragmentChecker fragmentChecker;
+ private final boolean proceedPastRep;
 
  public Suggester(Dictionary dictionary) {
- this(dictionary, null, FragmentChecker.EVERYTHING_POSSIBLE);
+ this(dictionary, null, FragmentChecker.EVERYTHING_POSSIBLE, false);
  }
 
  private Suggester(
- Dictionary dictionary, SuggestibleEntryCache suggestibleCache, FragmentChecker checker) {
+ Dictionary dictionary,
+ SuggestibleEntryCache suggestibleCache,
+ FragmentChecker checker,
+ boolean proceedPastRep) {
  this.dictionary = dictionary;
  this.suggestibleCache = suggestibleCache;
  this.fragmentChecker = checker;
+ this.proceedPastRep = proceedPastRep;
  }
 
  /**
@@ -71,16 +76,26 @@ private Suggester(
  * entries are stored as fast-to-iterate plain words instead of highly compressed prefix trees.
  */
  public Suggester withSuggestibleEntryCache() {
- return new Suggester(
-  dictionary, SuggestibleEntryCache.buildCache(dictionary.words), fragmentChecker);
+ SuggestibleEntryCache cache = SuggestibleEntryCache.buildCache(dictionary.words);
+ return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep);
  }
 
  /**
  * Returns a copy of this suggester instance with {@link FragmentChecker} hint that can improve
  * the performance of the "Modification" phase performance.
  */
  public Suggester withFragmentChecker(FragmentChecker checker) {
- return new Suggester(dictionary, suggestibleCache, checker);
+ return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep);
+ }
+
+ /**
+ * Returns a copy of this suggester instance that doesn't stop after encountering acceptable words
+ * after applying REP rules. By default, Hunspell stops when it finds any, but this behavior may
+ * not always be desirable, e.g., if we have "REP i ea", "tims" be replaced only by "teams" and
+ * not "times", which could also be meant.
+ */
+ public Suggester proceedPastRep() {
+ return new Suggester(dictionary, suggestibleCache, fragmentChecker, true);
  }
 
  /**
@@ -174,7 +189,8 @@ Root<CharsRef> findStem(
  }
 
  boolean hasGoodSuggestions =
- new ModifyingSuggester(suggestionSpeller, suggestions, word, wordCase, fragmentChecker)
+ new ModifyingSuggester(
+ suggestionSpeller, suggestions, word, wordCase, fragmentChecker, proceedPastRep)
  .suggest();
 
  if (!hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {

lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java

@@ -59,6 +59,14 @@ public void testAllcaps() throws Exception {
 
  public void testRepSuggestions() throws Exception {
  doTest("rep");
+
+ //noinspection DataFlowIssue
+ Path aff = Path.of(getClass().getResource("rep.aff").toURI());
+ Dictionary dictionary = TestAllDictionaries.loadDictionary(aff);
+ Suggester suggester = new Suggester(dictionary);
+ assertEquals(List.of("auto's"), suggester.suggestNoTimeout("autos", () -> {}));
+ assertEquals(
+ List.of("auto's", "auto"), suggester.proceedPastRep().suggestNoTimeout("autos", () -> {}));
  }
 
  public void testPhSuggestions() throws Exception {