update LazyPretrainDataset

Tim-taoxq · Jan 8, 2024 · f65243d · f65243d
1 parent a93978f
commit f65243d
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/component/dataset.py b/component/dataset.py
@@ -502,7 +502,7 @@ def __iter__(self):
  load_from_cache_file=True,
  keep_in_memory=False,
  cache_file_names={k: os.path.join(self.cache_dir, file_name, 'tokenized.arrow') for k in raw_dataset},
- desc=f"Tokenizing data: {file}",
+ desc=f"Tokenizing data",
  )
 
  # 拼接所有token