forked from ConnorJL/GPT2
-
Notifications
You must be signed in to change notification settings - Fork 0
/
117M.json
32 lines (32 loc) · 826 Bytes
/
117M.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
{
"n_head": 12,
"encoder_path": "gs:https://openwebtext/stuff/encoder",
"n_vocab": 50257,
"embed_dropout": 0.1,
"lr": 0.00025,
"warmup_steps": 2000,
"beta1": 0.9,
"beta2": 0.98,
"epsilon": 1e-9,
"opt_name": "adam",
"weight_decay": 0.01,
"train_batch_size": 32,
"attn_dropout": 0.1,
"train_steps": 10000,
"eval_steps": 10,
"max_steps": 500000,
"data_path": "gs:https://connors-datasets/openwebtext/",
"res_dropout": 0.1,
"predict_batch_size": 1,
"eval_batch_size": 32,
"iterations": 500,
"n_embd": 768,
"input": "openwebtext",
"model": "GPT2",
"model_path": "gs:https://connors-models/GPT2-117M-test-newinit",
"n_ctx": 1024,
"predict_path": "logs/predictions.txt",
"n_layer": 12,
"scale_by_depth": true,
"scale_by_in": true
}