Use tiktoken (openai#1044)

* use tiktoken==0.3.0

* formatting

* tuple should be safer

* Update whisper/tokenizer.py

Co-authored-by: Ruhollah Majdoddin <[email protected]>

* use tiktoken 0.3.1

* reflecting suggestions

* cleanup

* bypassing load_tiktoken_bpe to avoid blobfile dep

---------

Co-authored-by: Ruhollah Majdoddin <[email protected]>

MANIFEST.in

@@ -2,6 +2,4 @@ include requirements.txt
 include README.md
 include LICENSE
 include whisper/assets/*
-include whisper/assets/gpt2/*
-include whisper/assets/multilingual/*
 include whisper/normalizers/english.json

requirements.txt

@@ -3,5 +3,5 @@ numpy
 torch
 tqdm
 more-itertools
-transformers>=4.19.0
+tiktoken==0.3.1
 ffmpeg-python==0.2.0

tests/test_transcribe.py

@@ -4,6 +4,7 @@
 import torch
 
 import whisper
+from whisper.tokenizer import get_tokenizer
 
 
 @pytest.mark.parametrize("model_name", whisper.available_models())
@@ -24,14 +25,18 @@ def test_transcribe(model_name: str):
  assert "your country" in transcription
  assert "do for you" in transcription
 
+ tokenizer = get_tokenizer(model.is_multilingual)
+ all_tokens = [t for s in result["segments"] for t in s["tokens"]]
+ assert tokenizer.decode(all_tokens) == result["text"]
+ assert tokenizer.decode_with_timestamps(all_tokens).startswith("<|0.00|>")
+
  timing_checked = False
  for segment in result["segments"]:
  for timing in segment["words"]:
  assert timing["start"] < timing["end"]
  if timing["word"].strip(" ,") == "Americans":
  assert timing["start"] <= 1.8
  assert timing["end"] >= 1.8
- print(timing)
  timing_checked = True
 
  assert timing_checked