update default config values

EleutherAI · Apr 25, 2021 · 207caf6 · 207caf6
1 parent 39ded94
commit 207caf6
Show file tree

Hide file tree

Showing 9 changed files with 66 additions and 44 deletions.
diff --git a/.gitignore b/.gitignore
@@ -134,4 +134,6 @@ wandb/
 # data files 
 data/
 checkpoints/
-.vscode/
+.vscode/
+*.pt
+*.ckpt
diff --git a/configs/13B.yml b/configs/13B.yml
@@ -11,19 +11,21 @@
  "num-attention-heads": 40,
  "seq-length": 2048,
  "max-position-embeddings": 2048,
- "norm": "rmsnorm",
- "pos-emb": "none",
+ "norm": "layernorm",
+ "pos-emb": "rotary",
  "no-weight-tying": true,
- # this should provide some speedup but takes a while to build, set to true if desired
+
+ # these should provide some speedup but takes a while to build, set to true if desired
  "scaled-upper-triang-masked-softmax-fusion": false,
-
+ "bias-gelu-fusion": false,
+
  # optimizer settings
  "optimizer": {
  "type": "Adam",
  "params": {
  "lr": 0.0001,
- "max_grad_norm": 1.0,
- "betas": [0.9, 0.999]
+ "betas": [0.9, 0.999],
+ "eps": 1.0e-8,
  }
  },
  "zero_optimization": {

diff --git a/configs/175B.yml b/configs/175B.yml
@@ -11,19 +11,21 @@
  "num-attention-heads": 96,
  "seq-length": 2048,
  "max-position-embeddings": 2048,
- "norm": "rmsnorm",
- "pos-emb": "none",
+ "norm": "layernorm",
+ "pos-emb": "rotary",
  "no-weight-tying": true,
- # this should provide some speedup but takes a while to build, set to true if desired
+
+ # these should provide some speedup but takes a while to build, set to true if desired
  "scaled-upper-triang-masked-softmax-fusion": false,
-
+ "bias-gelu-fusion": false,
+
  # optimizer settings
  "optimizer": {
  "type": "Adam",
  "params": {
  "lr": 0.00006,
- "max_grad_norm": 1.0,
- "betas": [0.9, 0.999]
+ "betas": [0.9, 0.999],
+ "eps": 1.0e-8,
  }
  },
  "zero_optimization": {

diff --git a/configs/2-7B.yml b/configs/2-7B.yml
@@ -11,19 +11,22 @@
  "num-attention-heads": 32,
  "seq-length": 2048,
  "max-position-embeddings": 2048,
- "norm": "rmsnorm",
- "pos-emb": "none",
+ "norm": "layernorm",
+ "pos-emb": "rotary",
  "no-weight-tying": true,
- # this should provide some speedup but takes a while to build, set to true if desired
+
+ # these should provide some speedup but takes a while to build, set to true if desired
  "scaled-upper-triang-masked-softmax-fusion": false,
+ "bias-gelu-fusion": false,
+
 
  # optimizer settings
  "optimizer": {
  "type": "Adam",
  "params": {
  "lr": 0.00016,
- "max_grad_norm": 1.0,
- "betas": [0.9, 0.999]
+ "betas": [0.9, 0.999],
+ "eps": 1.0e-8,
  }
  },
  "zero_optimization": {

diff --git a/configs/6-7B.yml b/configs/6-7B.yml
@@ -11,19 +11,22 @@
  "num-attention-heads": 32,
  "seq-length": 2048,
  "max-position-embeddings": 2048,
- "norm": "rmsnorm",
- "pos-emb": "none",
+ "norm": "layernorm",
+ "pos-emb": "rotary",
  "no-weight-tying": true,
- # this should provide some speedup but takes a while to build, set to true if desired
+
+ # these should provide some speedup but takes a while to build, set to true if desired
  "scaled-upper-triang-masked-softmax-fusion": false,
+ "bias-gelu-fusion": false,
+
 
  # optimizer settings
  "optimizer": {
  "type": "Adam",
  "params": {
  "lr": 0.00012,
- "max_grad_norm": 1.0,
- "betas": [0.9, 0.999]
+ "betas": [0.9, 0.999],
+ "eps": 1.0e-8,
  }
  },
  "zero_optimization": {

diff --git a/configs/XL.yml b/configs/XL.yml
@@ -11,20 +11,21 @@
  "num-attention-heads": 16,
  "seq-length": 2048,
  "max-position-embeddings": 2048,
- "norm": "rmsnorm",
- "pos-emb": "none",
+ "norm": "layernorm",
+ "pos-emb": "rotary",
  "no-weight-tying": true,
- # this should provide some speedup but takes a while to build, set to true if desired
+
+ # these should provide some speedup but takes a while to build, set to true if desired
  "scaled-upper-triang-masked-softmax-fusion": false,
+ "bias-gelu-fusion": false,
 
-
  # optimizer settings
  "optimizer": {
  "type": "Adam",
  "params": {
  "lr": 0.0002,
- "max_grad_norm": 1.0,
- "betas": [0.9, 0.999]
+ "betas": [0.9, 0.999],
+ "eps":  1.0e-8,
  }
  },
  "zero_optimization": {

diff --git a/configs/large.yml b/configs/large.yml
@@ -11,19 +11,22 @@
  "num-attention-heads": 16,
  "seq-length": 2048,
  "max-position-embeddings": 2048,
- "norm": "rmsnorm",
- "pos-emb": "none",
+ "norm": "layernorm",
+ "pos-emb": "rotary",
  "no-weight-tying": true,
- # this should provide some speedup but takes a while to build, set to true if desired
+
+ # these should provide some speedup but takes a while to build, set to true if desired
  "scaled-upper-triang-masked-softmax-fusion": false,
+ "bias-gelu-fusion": false,
+
 
  # optimizer settings
  "optimizer": {
  "type": "Adam",
  "params": {
  "lr": 0.00025,
- "max_grad_norm": 1.0,
- "betas": [0.9, 0.999]
+ "betas": [0.9, 0.999],
+ "eps": 1.0e-8,
  }
  },
  "zero_optimization": {

diff --git a/configs/medium.yml b/configs/medium.yml
@@ -11,20 +11,23 @@
  "num-attention-heads": 16,
  "seq-length": 2048,
  "max-position-embeddings": 2048,
- "norm": "rmsnorm",
- "pos-emb": "none",
+ "norm": "layernorm",
+ "pos-emb": "rotary",
  "no-weight-tying": true,
- # this should provide some speedup but takes a while to build, set to true if desired
+
+ # these should provide some speedup but takes a while to build, set to true if desired
  "scaled-upper-triang-masked-softmax-fusion": false,
+ "bias-gelu-fusion": false,
+
 
 
  # optimizer settings
  "optimizer": {
  "type": "Adam",
  "params": {
  "lr": 0.0003,
- "max_grad_norm": 1.0,
- "betas": [0.9, 0.999]
+ "betas": [0.9, 0.999],
+ "eps": 1.0e-8,
  }
  },
  "zero_optimization": {

diff --git a/configs/small.yml b/configs/small.yml
@@ -11,19 +11,22 @@
  "num-attention-heads": 12,
  "seq-length": 2048,
  "max-position-embeddings": 2048,
- "norm": "rmsnorm",
- "pos-emb": "none",
+ "norm": "layernorm",
+ "pos-emb": "rotary",
  "no-weight-tying": true,
- # this should provide some speedup but takes a while to build, set to true if desired
+
+ # these should provide some speedup but takes a while to build, set to true if desired
  "scaled-upper-triang-masked-softmax-fusion": false,
+ "bias-gelu-fusion": false,
+
 
  # optimizer settings
  "optimizer": {
  "type": "Adam",
  "params": {
  "lr": 0.0006,
- "max_grad_norm": 1.0,
- "betas": [0.9, 0.999]
+ "betas": [0.9, 0.999],
+ "eps": 1.0e-8,
  }
  },
  "zero_optimization": {