improve finetune compatibility

josStorer · Jul 3, 2023 · 511652b · 511652b
1 parent 525fb13
commit 511652b
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/finetune/json2binidx_tool/tools/preprocess_data.py b/finetune/json2binidx_tool/tools/preprocess_data.py
@@ -17,11 +17,14 @@
 
 """Processing data for pretraining."""
 
-import argparse
-import multiprocessing
 import os
 import sys
 
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+
+import argparse
+import multiprocessing
+
 import lm_dataformat as lmd
 import numpy as np
 

diff --git a/main.go b/main.go
@@ -37,6 +37,7 @@ func main() {
  backend.CopyEmbed(finetune)
  os.Mkdir("models", os.ModePerm)
  os.Mkdir("lora-models", os.ModePerm)
+ os.Mkdir("finetune/json2binidx_tool/data", os.ModePerm)
  }
 
  f, err := os.Create("lora-models/train_log.txt")