Update BERT tokenizer integration to reflect API changes in Transform…

…ers library (#42) * Be more specific about the version of the Transformers library * Update BERT tokenizer integration to reflect latest APIs of the Transformers library
CODAIT · Jul 2, 2020 · 8abb7b6 · 8abb7b6
1 parent 6819c52
commit 8abb7b6
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/config/dev_reqs.txt b/config/dev_reqs.txt
@@ -4,7 +4,7 @@
 # NOTE: These requirements are IN ADDITION TO the packages in
 # config/dev_env.yml and requirements.txt
 pyyaml
-transformers
+transformers>=3.0.0
 spacy
 fastparquet
 sphinx

diff --git a/text_extensions_for_pandas/io/tokenization.py b/text_extensions_for_pandas/io/tokenization.py
@@ -49,7 +49,7 @@ def make_bert_tokens(target_text: str, tokenizer) -> pd.DataFrame:
  * "special_tokens_mask": `True` if the token is a zero-length special token
  such as "start of document"
  """
- from transformers.tokenization_utils import PreTrainedTokenizerFast
+ from transformers import PreTrainedTokenizerFast
 
  if not isinstance(tokenizer, PreTrainedTokenizerFast):
  raise TypeError(