diff --git a/evals/registry/data/icelandic-sentences-gec/samples.jsonl b/evals/registry/data/icelandic-sentences-gec/samples.jsonl new file mode 100644 index 0000000000..d9d5097a3f --- /dev/null +++ b/evals/registry/data/icelandic-sentences-gec/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e8f63f51c5d5e80f48a8bfe4a4785decbb915174a9ea02763aec66af5f4c189 +size 126174 diff --git a/evals/registry/evals/icelandic-sentences-gec.yaml b/evals/registry/evals/icelandic-sentences-gec.yaml new file mode 100644 index 0000000000..b0300c1ebb --- /dev/null +++ b/evals/registry/evals/icelandic-sentences-gec.yaml @@ -0,0 +1,9 @@ +icelandic-sentences-gec: + id: icelandic-sentences-gec.dev.v0 + description: Test the model's ability to correctly identify whether an Icelandic sentence contains a grammatical error. + metrics: [accuracy] + +icelandic-sentences-gec.dev.v0: + class: evals.elsuite.basic.match:Match + args: + samples_jsonl: icelandic-sentences-gec/samples.jsonl