[evals] added more abc options to from_n choice_strings (openai#404)

gauravjaincr7 · Mar 22, 2023 · 8294f6c · 8294f6c
1 parent 4f090a0
commit 8294f6c
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/evals/elsuite/modelgraded/classify.py b/evals/elsuite/modelgraded/classify.py
@@ -136,6 +136,10 @@ def __init__(
  self.choice_strings = modelgraded_specs.pop("choice_strings")
  if self.choice_strings == "from_n":
  self.choice_strings = [str(i + 1) for i in range(self.multicomp_n)]
+ elif self.choice_strings == "from_n_abc":
+ self.choice_strings = [string.ascii_lowercase[i] for i in range(self.multicomp_n)]
+ elif self.choice_strings == "from_n_ABC":
+ self.choice_strings = [string.ascii_uppercase[i] for i in range(self.multicomp_n)]
  # make sure each choice doesn't contain any punctuation
  for s in self.choice_strings:
  assert not any(c in s for c in string.punctuation), f"{s} contains punctuation"