添加数据集处理进度

Lix1993 · Nov 21, 2021 · a976086 · a976086
1 parent 1c718f7
commit a976086
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/data_helper.py b/data_helper.py
@@ -96,10 +96,10 @@ def data_process(self, filepath):
  :return:
  """
 
- raw_iter = iter(open(filepath, encoding="utf8"))
+ raw_iter = open(filepath,encoding='utf8').readlines()
  data = []
  max_len = 0
- for raw in tqdm(raw_iter):
+ for raw in tqdm(raw_iter, ncols=80):
  line = raw.rstrip("\n").split('","')
  s, l = line[-1][:-1], line[0][1:]
  s = clean_str(s)