Added t-few configs.

clinicalml · Apr 12, 2023 · a729f79 · a729f79
1 parent 4487a4e
commit a729f79
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 0 deletions.
diff --git a/t-few/configs/global.json b/t-few/configs/global.json
@@ -0,0 +1,11 @@
+{
+ "batch_size": 8,
+ "eval_batch_size": 16,
+ "grad_accum_factor": 1,
+ "num_shot": 32,
+ "length_norm": 1,
+ "mc_loss": 1,
+ "unlikely_loss": 1,
+
+ "max_seq_len": 1024
+}
diff --git a/t-few/configs/ia3.json b/t-few/configs/ia3.json
@@ -0,0 +1,11 @@
+{
+ "lora_scaling_rank": 1,
+ "lora_rank": 0,
+ "lora_init_scale": 0.0,
+ "lora_modules": ".*SelfAttention|.*EncDecAttention|.*DenseReluDense",
+ "lora_layers": "k|v|wi_1.*",
+ "trainable_param_names": ".*lora_b.*",
+ "model_modifier": "lora",
+ "lr": 3e-3,
+ "num_steps": 1000
+}
diff --git a/t-few/configs/t011b.json b/t-few/configs/t011b.json
@@ -0,0 +1,7 @@
+{
+ "exp_name": "debug",
+ "origin_model": "bigscience/T0",
+ "compute_precision": "bf16",
+ "compute_strategy": "none",
+ "allow_skip_exp":true
+}
diff --git a/t-few/configs/t03b.json b/t-few/configs/t03b.json
@@ -0,0 +1,7 @@
+{
+ "exp_name": "debug",
+ "origin_model": "bigscience/T0_3B",
+ "compute_precision": "bf16",
+ "compute_strategy": "none",
+ "allow_skip_exp":true
+}