[evals] support abc/ABC for choice_template (openai#406)

gauravjaincr7 · Mar 23, 2023 · 4aba959 · 4aba959
1 parent 8a4c158
commit 4aba959
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/evals/elsuite/modelgraded/classify.py b/evals/elsuite/modelgraded/classify.py
@@ -137,9 +137,9 @@ def __init__(
         if self.choice_strings == "from_n":
             self.choice_strings = [str(i + 1) for i in range(self.multicomp_n)]
         elif self.choice_strings == "from_n_abc":
-            self.choice_strings = [string.ascii_lowercase[i] for i in range(self.multicomp_n)]
+            self.choice_strings = [string.ascii_lowercase[i % 26] for i in range(self.multicomp_n)]
         elif self.choice_strings == "from_n_ABC":
-            self.choice_strings = [string.ascii_uppercase[i] for i in range(self.multicomp_n)]
+            self.choice_strings = [string.ascii_uppercase[i % 26] for i in range(self.multicomp_n)]
         # make sure each choice doesn't contain any punctuation
         for s in self.choice_strings:
             assert not any(c in s for c in string.punctuation), f"{s} contains punctuation"
@@ -259,6 +259,8 @@ def eval_sample(self, test_sample: dict, rng: Random) -> None:
                             completion += format_necessary(
                                 completion_i_template,
                                 i=i + 1,
+                                i_abc=string.ascii_lowercase[i % 26],
+                                i_ABC=string.ascii_uppercase[i % 26],
                                 output=completion_i,
                                 n=self.multicomp_n,
                             )