Merge pull request EleutherAI#806 from EleutherAI/tiktoken_import_fix

Only import tiktoken if we intend to use it
hisashi-ito · Mar 3, 2023 · 192022f · 192022f
2 parents 60cac3a + e8a4684
commit 192022f
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/configs/neox_arguments.md b/configs/neox_arguments.md
@@ -111,7 +111,7 @@ Logging Arguments
 
 - **git_hash**: str
 
- Default = a7febb2
+ Default = 496f987
 
  current git hash of repository
 

diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py
@@ -24,7 +24,6 @@
 from transformers import GPT2Tokenizer, GPT2TokenizerFast
 import numpy as np
 import sentencepiece as spm
-import tiktoken
 from typing import List, Union
 from .gpt2_tokenization import GPT2Tokenizer
 
@@ -353,6 +352,11 @@ def eod(self):
 
 class TiktokenTokenizer(AbstractTokenizer):
  """Tokenizer from OpenAI's tiktoken implementation"""
+ try:
+ import tiktoken
+ except ModuleNotFoundError:
+ print("Please install tiktoken: (https://github.com/openai/tiktoken)")
+ raise Exception
 
  def __init__(self, vocab_file):
  name = "TiktokenTokenizer"