Update tokenizer.py

EleutherAI · Mar 9, 2023 · 50d3e6c · 50d3e6c
1 parent ad7af96
commit 50d3e6c
Showing 1 changed file with 5 additions and 5 deletions.
diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py
@@ -352,11 +352,11 @@ def eod(self):
 
 class TiktokenTokenizer(AbstractTokenizer):
  """Tokenizer from OpenAI's tiktoken implementation"""
-  try:
-  import tiktoken
-  except ModuleNotFoundError:
-  print("Please install tiktoken: (https://github.com/openai/tiktoken)")
-  raise Exception
+ try:
+ import tiktoken
+ except ModuleNotFoundError:
+ print("Please install tiktoken: (https://github.com/openai/tiktoken)")
+ raise Exception
 
  def __init__(self, vocab_file):
  name = "TiktokenTokenizer"