change onebitadam to no zero

EleutherAI · Feb 12, 2021 · b5a4a7d · b5a4a7d
1 parent 84a4618
commit b5a4a7d
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/configs/deepspeed_configs/ds_zero_stage_1_onebitadam.json b/configs/deepspeed_configs/ds_zero_stage_1_onebitadam.json
@@ -3,7 +3,7 @@
  "train_micro_batch_size_per_gpu": 4,
  "steps_per_print": 10,
  "zero_optimization": {
- "stage": 1,
+ "stage": 0,
  "allgather_partitions": true,
  "reduce_scatter": true,
  "allgather_bucket_size": 50000000,

diff --git a/examples/ds_pretrain_gpt2_XL_pipe_onebit.sh b/examples/ds_pretrain_gpt2_XL_pipe_onebit.sh
@@ -35,7 +35,7 @@ GAS=16
 LOGDIR="tensorboard_data/${NLAYERS}l_${NHIDDEN}h_${NNODES}n_${GPUS_PER_NODE}g_${pp_size}pp_${mp_size}mp_${BATCHSIZE}b_ds4"
 
 #ZeRO Configs
-stage=1
+stage=0
 reduce_scatter=true
 contigious_gradients=true
 rbs=50000000
@@ -50,7 +50,7 @@ SYNCHRONIZE=true
 PROFILE=false
 
 # GPT options:
-NLAYERS=24
+NLAYERS=12
 NHIDDEN=1024
 NHEADS=16
 SEQLEN=1024