Update gpt2_dataset.py (#974)

* Update gpt2_dataset.py * Update NeoXArgs docs automatically --------- Co-authored-by: github-actions <[email protected]>
EleutherAI · Jun 22, 2023 · 2534e3d · 2534e3d
1 parent cfce548
commit 2534e3d
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/configs/neox_arguments.md b/configs/neox_arguments.md
@@ -111,7 +111,7 @@ Logging Arguments
 
 - **git_hash**: str
 
- Default = 07da9fc
+ Default = 6b22c39
 
  current git hash of repository
 
@@ -1061,11 +1061,12 @@ Training Arguments
  List of paths to train datasets.
 
 
+
 - **label_data_paths**: list
 
  Default = None
 
- List of paths to label datasets (should be fully in sync with train data, not shifted by 1!).
+ List of paths to label datasets (not shifted by 1 yet!).
 
 
 

diff --git a/megatron/data/gpt2_dataset.py b/megatron/data/gpt2_dataset.py
@@ -64,7 +64,7 @@ def __init__(
  self.shuffle_idx_len = self.shuffle_idx.shape[0] - 1
  self.sample_idx_len = self.sample_idx.shape[0] - 1
 
- if self.shuffle_idx_len != self.sample_idx_len:
+ if self.shuffle_idx_len != self.sample_idx_len - 1:
  print(
  f"WARNING: shuffle index length ({self.shuffle_idx_len}) is not equal to sample index length ({self.sample_idx_len})"
  )