fix tokenizer (mikecovlee#43)

mlora/tokenizer.py

@@ -15,14 +15,20 @@ def __init__(self, model_path: str):
         self.pad_id_ = self.tokenizer.pad_token_id
         self.unk_id_ = self.tokenizer.unk_token_id
         # maybe pad id is unk
-        if self.pad_id_ is None and self.unk_id_ is not None:
+        if self.pad_id_ is not None:
+            return
+        if self.unk_id_ is None and self.eos_id_ is None:
+            logging.warn(
+                "Detecting <eos> and <unk> are None, setting <pad> to 0 by default.")
+            self.pad_id_ = 0
+        elif self.unk_id_ is not None:
+            logging.warn(
+                "Detecting <pad> is None, setting to <unk> by default.")
             self.pad_id_ = self.unk_id_
-        if self.pad_id_ is None and self.eos_id_ is not None:
+        elif self.eos_id_ is not None:
+            logging.warn(
+                "Detecting <pad> is None, setting to <eos> by default.")
             self.pad_id_ = self.eos_id_
-            logging.warn("Padding token ID is None, setting to <eos>.")
-        else:
-            raise ValueError(
-                "Can not set padding token id. <eos> and <unk> are None.")
 
     def encode(self, data: Union[str, List[str]], add_special_tokens: bool = True) -> Tokens:
         if isinstance(data, str):

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mlora"
-version = "0.3.0.post1"
+version = "0.3.0.post2"
 description = "A tool for fine-tuning large language models (LLMs) using the LoRA or QLoRA methods more efficiently."
 readme = "README.md"
 requires-python = ">=3.10"