-
Notifications
You must be signed in to change notification settings - Fork 6
/
run_streaming.sh
31 lines (29 loc) · 829 Bytes
/
run_streaming.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# improvement: huge
export LCKV_FLASH_ATTN=1
# improvement: significant
export LCKV_FUSED_RMSNORM=1
# improvement: none
export LCKV_FUSED_CROSSENTROPY=1
# improvement: none
export LCKV_FUSED_ROTARY=1
# improvement: slightly
export LCKV_FUSED_SWIGLU=1
# streaming test
python test_streaming.py \
--model_name_or_path outputs/llamatiny-3090-test \
--output_dir streaming/llamatiny-3090-test \
--dataset_name pg19 \
--num_eval_tokens 4000000 \
--enable_streaming \
--start_size 4 \
--recent_size 1020 \
# performance test
python test_streaming.py \
--model_name_or_path outputs/llamatiny-3090-test \
--output_dir streaming/llamatiny-2048 \
--dataset_name pg19 \
--download_streaming \
--num_eval_tokens 65133 \
--enable_streaming \
--start_size 4 \
--recent_size 2044 \