add descriptive errors for unimplemented class methods

jamestiotio · Jan 31, 2023 · 3463ca4 · 3463ca4
1 parent 9879ef2
commit 3463ca4
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py
@@ -368,11 +368,12 @@ def vocab_size(self):
 
  @property
  def vocab(self):
- raise NotImplementedError
+ raise NotImplementedError("TiktokenTokenizer does not implement vocabulary access.")
 
  @property
  def inv_vocab(self):
- raise NotImplementedError
+ raise NotImplementedError("TiktokenTokenizer does not implement vocabulary access. \
+ To get the idx-th token in vocabulary, use tokenizer.decode([idx]) .")
 
  def tokenize(self, text: str):
  return self.tokenizer.encode(text) #, allowed_special="all")