Try using non-transformer spacy model

CouncilDataProject · Dec 5, 2023 · 623871b · 623871b
1 parent 06f618a
commit 623871b
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 5 deletions.
diff --git a/cdp_backend/sr_models/whisper.py b/cdp_backend/sr_models/whisper.py
@@ -9,7 +9,6 @@
 from typing import TYPE_CHECKING, Any
 
 import spacy
-import spacy_transformers # noqa: F401
 from faster_whisper import WhisperModel as FasterWhisper
 from pydub import AudioSegment
 from spacy.cli.download import download as download_spacy_model
@@ -30,6 +29,7 @@
 
 ###############################################################################
 
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 spacy.prefer_gpu()
 
 ###############################################################################
@@ -48,7 +48,7 @@ class WhisperModel(SRModel):
  @staticmethod
  def _load_spacy_model() -> "Language":
  nlp = spacy.load(
- "en_core_web_trf",
+ DEFAULT_SPACY_MODEL,
  # Only keep the parser
  # We are only using this for sentence parsing
  disable=[
@@ -127,7 +127,7 @@ def __init__(
  try:
  self.nlp = self._load_spacy_model()
  except Exception:
- download_spacy_model("en_core_web_trf")
+ download_spacy_model(DEFAULT_SPACY_MODEL)
  self.nlp = self._load_spacy_model()
 
  def transcribe(

diff --git a/pyproject.toml b/pyproject.toml
@@ -57,7 +57,6 @@ Documentation = "https://CouncilDataProject.github.io/cdp-backend"
 # https://peps.python.org/pep-0621/#dependencies-optional-dependencies
 [project.optional-dependencies]
 pipeline = [
- "curated-transformers>=0.1.1",
  "dask[distributed]>=2021.7.0",
  "faster-whisper==0.3.0",
  "ffmpeg-python>=0.2.0",
@@ -74,7 +73,6 @@ pipeline = [
  "pypdf>=2.0",
  "rapidfuzz>=2.0",
  "spacy>=3.4",
- "spacy-transformers>=1.1",
  "tika==2.6.0",
  "torch>=1.10,<2",
  "tqdm>=4.62",