1.5B.json

{
    "n_head": 25,
    "encoder_path": "gs:https://openwebtext/stuff/encoder",
    "n_vocab": 50257,
    "embed_dropout": 0.1,
    "lr": 0.00025,
    "warmup_steps": 2000,
    "beta1": 0.0,
    "decay_exponent": 0.8,
    "opt_name": "adafactor",
    "decay_type": "pow",
    "train_batch_size": 512,
    "attn_dropout": 0.1,
    "train_steps": 100000,
    "eval_steps": 10,
    "max_steps": 300000,
    "data_path": "gs:https://connors-datasets/openwebtext/",
    "res_dropout": 0.1,
    "predict_batch_size": 1,
    "eval_batch_size": 8,
    "iterations": 100,
    "n_embd": 1600,
    "input": "openwebtext_longbiased",
    "model": "GPT2",
    "model_path": "gs:https://connors-models/1.5B",
    "n_ctx": 1024,
    "predict_path": "logs/predictions_1.5B.txt",
    "n_layer": 48,
    "precision": "float32",
    "weight_decay": 0.01,
    "scale_by_depth": true,
    "scale_by_in": true
}