tokenizer update

hisashi-ito · Mar 12, 2023 · 802b6fe · 802b6fe
1 parent a188bb1
commit 802b6fe
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
@@ -11,6 +11,8 @@ regex
 sentencepiece
 six
 tiktoken==0.1.2
-tokenizers==0.12.1
+# 2023.03.12 update
+# tokenizers==0.12.1
+tokenizers==0.13.2
 transformers~=4.24.0
 wandb==0.10.28