added init method std

zTaoplus · Jan 27, 2021 · a6bf1a0 · a6bf1a0
1 parent 60704e7
commit a6bf1a0
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/examples/pretrain_gpt3_175B.sh b/examples/pretrain_gpt3_175B.sh
@@ -46,6 +46,7 @@ options=" \
  --weight-decay 0.1 \
  --adam-beta1 0.9 \
  --adam-beta2 0.95 \
+ --init-method-std 0.006 \
  --tensorboard-dir <TENSORBOARD DIRECTORY> \
  --fp16 \
  --checkpoint-activations "