Add in Reverse String eval

lucemia · Mar 14, 2023 · 357039b · 357039b
1 parent 38eb92c
commit 357039b
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 0 deletions.
diff --git a/evals/registry/data/reverse_string/reverse_string.jsonl b/evals/registry/data/reverse_string/reverse_string.jsonl
diff --git a/evals/registry/evals/reverse-string.yaml b/evals/registry/evals/reverse-string.yaml
@@ -0,0 +1,9 @@
+reverse-string:
+ id: reverse-string.s1.simple-v0
+ description: Test the model's ability to reverse complex and simple strings.
+ disclaimer: The eval might be off due to model output being in unicode formate vs raw text.
+ metrics: [accuracy]
+reverse-string.s1.simple-v0:
+ class: evals.elsuite.basic.match:Match
+ args:
+ samples_jsonl: reverse_string/reverse_string.jsonl