diff --git a/evals/registry/data/rucola/few_shot.jsonl b/evals/registry/data/rucola/few_shot.jsonl new file mode 100644 index 0000000000..4f2e5a9eed --- /dev/null +++ b/evals/registry/data/rucola/few_shot.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f202493d308ff562c89fe2600a93d03fa1bda76bd37e9594836e6eface3a799 +size 2746 diff --git a/evals/registry/data/rucola/samples.jsonl b/evals/registry/data/rucola/samples.jsonl new file mode 100644 index 0000000000..906e71b20b --- /dev/null +++ b/evals/registry/data/rucola/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:264f2e85219deaefc592f3b3e7825101b050a1fa44325e8f35590f6e565187b0 +size 75375 diff --git a/evals/registry/evals/rucola.yaml b/evals/registry/evals/rucola.yaml new file mode 100644 index 0000000000..768fc2ba73 --- /dev/null +++ b/evals/registry/evals/rucola.yaml @@ -0,0 +1,11 @@ +rucola: + id: rucola.test.v0 + description: . + metrics: [accuracy] + +rucola.test.v0: + class: evals.elsuite.basic.match:Match + args: + samples_jsonl: rucola/samples.jsonl + few_shot_jsonl: rucola/few_shot.jsonl + num_few_shot: 4