-
Notifications
You must be signed in to change notification settings - Fork 181
/
7B.yaml
36 lines (30 loc) · 697 Bytes
/
7B.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# data
data:
instruct_data: "" # Fill
data: "" # Optionally fill with pretraining data
eval_instruct_data: "" # Optionally fill
# model
model_id_or_path: "" # Change to downloaded path
lora:
rank: 64
# optim
seq_len: 32768
batch_size: 1
max_steps: 300
optim:
lr: 6.e-5
weight_decay: 0.1
pct_start: 0.05
# other
seed: 0
log_freq: 1
eval_freq: 100
no_eval: False
ckpt_freq: 100
save_adapters: True # save only trained LoRA adapters. Set to `False` to merge LoRA adapter into the base model and save full fine-tuned model
run_dir: "" # Fill
wandb:
project: "" # your wandb project name
run_name: "" # your wandb run name
key: "" # your wandb api key
offline: False