Remove gas (beano) (#1144)

* Remove 'gas' configuration variable * Remove gas from configs and config documentation * Update training.py
EleutherAI · Feb 5, 2024 · 91c44bc · 91c44bc
1 parent 032ec8c
commit 91c44bc
Show file tree

Hide file tree

Showing 8 changed files with 8 additions and 21 deletions.
diff --git a/configs/19M.yml b/configs/19M.yml
@@ -44,7 +44,7 @@
  },
 
  "train_micro_batch_size_per_gpu": 4, #32,
- "gas": 1,
+ "gradient_accumulation_steps": 1,
  "data_impl": "mmap",
  "num_workers": 1,
 

diff --git a/configs/49M.yml b/configs/49M.yml
@@ -49,7 +49,7 @@
 
  # batch / data settings
  "train_micro_batch_size_per_gpu": 32,
- "gas": 1,
+ "gradient_accumulation_steps": 1,
  "data_impl": "mmap",
  "num_workers": 1,
 

diff --git a/configs/800M.yml b/configs/800M.yml
@@ -44,7 +44,7 @@
  },
 
  "train_micro_batch_size_per_gpu": 16,
- "gas": 1,
+ "gradient_accumulation_steps": 1,
  "data_impl": "mmap",
  "num_workers": 1,
 

diff --git a/configs/neox_arguments.md b/configs/neox_arguments.md
@@ -976,7 +976,7 @@ Text Generation arguments
 
 - **prompt_end**: str
 
- Default = 
+ Default =
 
 
  a single prompt's end. Defaults to newline
@@ -1018,7 +1018,7 @@ Text Generation arguments
 
 - **eval_results_prefix**: str
 
- Default = 
+ Default =
 
  prefix to which to save evaluation results - final fp will be {eval_results_prefix}_eval_results_yy-mm-dd-HH-MM.json
 
@@ -1478,14 +1478,6 @@ Training Arguments
 
 
 
-- **gas**: int
-
- Default = None
-
- gradient_accumulation_steps
-
-
-
 - **clip_grad**: float
 
  Default = 1.0
@@ -1770,7 +1762,7 @@ Args for deepspeed config
 
  Default = None
 
- 
+
 
 
 
@@ -2070,4 +2062,3 @@ Args for deepspeed runner (deepspeed.launcher.runner).
  Default = None
 
  Adds a `--account` to the DeepSpeed launch command. In DeeperSpeed this is passed on to the SlurmLauncher as well. Sometimes necessary for cluster rules, or so I've heard.
-
diff --git a/megatron/neox_arguments/arguments.py b/megatron/neox_arguments/arguments.py
@@ -896,7 +896,6 @@ def calculate_derived(self):
  "gradient_accumulation_steps": gradient_accumulation_steps,
  "batch_size": train_micro_batch_size_per_gpu,
  # duplicate items
- "gas": self.gradient_accumulation_steps,
  "clip_grad": self.gradient_clipping,
  }
  )

diff --git a/megatron/neox_arguments/neox_args.py b/megatron/neox_arguments/neox_args.py
@@ -996,9 +996,6 @@ class NeoXArgsTraining(NeoXArgsTemplate):
  Partition Activations across GPUs before checkpointing.
  """
 
- gas: int = None
- """gradient_accumulation_steps""" # TODO this is a duplicate, remove?
-
  clip_grad: float = 1.0
  """
  Gradient clipping based on global L2 norm.

diff --git a/megatron/training.py b/megatron/training.py
@@ -927,7 +927,7 @@ def evaluate(
 
  # although we're not accumulating gradients here, we count one iter as train_batch_size_per_gpu * g.a.s
  # to be consistent with deepspeed's pipe parallel engine
- # since pipe parallel already takes gas into account - default to 1 here if pipe parallel is true
+ # since pipe parallel already takes gradient_accumulation_steps into account - default to 1 here if pipe parallel is true
  for _ in range(
  1
  if neox_args.is_pipe_parallel

diff --git a/tests/config/test_setup.yml b/tests/config/test_setup.yml
@@ -44,7 +44,7 @@
  "output_layer_init_method": "wang_init",
 
  "train_micro_batch_size_per_gpu": 4,
- "gas": 1,
+ "gradient_accumulation_steps": 1,
  "data_impl": "mmap",
  "num_workers": 1,