Merge pull request #813 from EleutherAI/fix-tok-import

fix unexpected indent error for conditional `tiktoken` import
EleutherAI · Mar 8, 2023 · 71df4d5 · 71df4d5
2 parents a5c2229 + 019da08
commit 71df4d5
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/configs/neox_arguments.md b/configs/neox_arguments.md
@@ -111,7 +111,7 @@ Logging Arguments
 
 - **git_hash**: str
 
- Default = 496f987
+ Default = 5face7a
 
  current git hash of repository
 

diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py
@@ -352,11 +352,11 @@ def eod(self):
 
 class TiktokenTokenizer(AbstractTokenizer):
  """Tokenizer from OpenAI's tiktoken implementation"""
-  try:
-  import tiktoken
-  except ModuleNotFoundError:
-  print("Please install tiktoken: (https://github.com/openai/tiktoken)")
-  raise Exception
+ try:
+ import tiktoken
+ except ModuleNotFoundError:
+ print("Please install tiktoken: (https://github.com/openai/tiktoken)")
+ raise Exception
 
  def __init__(self, vocab_file):
  name = "TiktokenTokenizer"