Utils: Fix language detection

BLKSerene · Jun 12, 2024 · 7089ce2 · 7089ce2
1 parent 9a28747
commit 7089ce2
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -45,6 +45,7 @@
 ### 📌 Bugfixes
 - File Area: Fix support for .xlsx files
 - Utils: Fix downloading of Stanza models
+- Utils: Fix language detection
 - Work Area: Fix Dependency Parser - analysis of files whose first token is a punctuation mark
 
 ### ❌ Removals

diff --git a/tests/tests_utils/test_detection.py b/tests/tests_utils/test_detection.py
@@ -298,15 +298,13 @@ def test_lingua():
  re.search(r'^[^\(\)]+', lang.lower()).group().strip()
  for lang in main.settings_global['langs']
  }
- langs_exceptions = {'bokmal', 'ganda', 'nynorsk', 'slovene'}
+ langs_exceptions = {'bokmal', 'ganda', 'nynorsk'}
  langs_extra = set()
 
  for lang in lingua.Language.all(): # pylint: disable=no-member
  if lang.name.lower() not in langs | langs_exceptions:
  langs_extra.add(lang.name)
 
- print(f"\nExtra languages: {', '.join(langs_extra)}\n")
-
  assert langs_extra == {'BOSNIAN', 'MAORI', 'SHONA', 'SOMALI', 'SOTHO', 'TSONGA', 'XHOSA'}
 
 def test_detection_lang():

diff --git a/wordless/wl_utils/wl_detection.py b/wordless/wl_utils/wl_detection.py
@@ -60,6 +60,8 @@ def detect_encoding(main, file_path):
  lingua.Language.BOSNIAN,
  lingua.Language.MAORI,
  lingua.Language.SHONA,
+ lingua.Language.SOMALI,
+ lingua.Language.SOTHO,
  lingua.Language.TSONGA,
  lingua.Language.XHOSA
 ).build()