Merge pull request #32 from wellecks/multiple-choice-fix

Multiple-choice-fix
wellecks · Oct 12, 2023 · 8ba804b · 8ba804b
2 parents 2ca15c3 + 36f350f
commit 8ba804b
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 4 deletions.
diff --git a/lm_eval/tasks/hendrycks_test_cot.py b/lm_eval/tasks/hendrycks_test_cot.py
@@ -149,7 +149,7 @@ class MinervaCoTMMLU(MajorityVotingMixin, Task):
     DATASET_PATH = "hendrycks_test"
     DATASET_NAME = None
 
-    ANS_RE = re.compile(r"Final Answer: The final answer is \([ABCD]\). I hope it is correct.")
+    ANS_RE = re.compile(r"Final Answer: The final answer is \([ABCD]\).")
     INVALID_ANS = "[not found]"
 
     def __init__(self, subject):
@@ -206,7 +206,7 @@ def doc_to_text(self, doc):
 
     @property
     def end_seq(self):
-        return ["\n\n", "Problem:"]
+        return ["I hope it is correct."]
 
     def process_results(self, doc, results, params={}):
         """Take a single document and the LM results and evaluates, returning a

diff --git a/lm_eval/tasks/math_sat_cot.py b/lm_eval/tasks/math_sat_cot.py
@@ -80,7 +80,7 @@ class MathSATCoT(MajorityVotingMixin, Task):
     DATASET_PATH = "mcaleste/sat_multiple_choice_math_may_23"
     DATASET_NAME = None
 
-    ANS_RE = re.compile(r"Final Answer: The final answer is \([ABCD]\). I hope it is correct.")
+    ANS_RE = re.compile(r"Final Answer: The final answer is \([ABCD]\).")
     INVALID_ANS = "[not found]"
 
     def __init__(self):
@@ -134,7 +134,8 @@ def doc_to_text(self, doc):
 
     @property
     def end_seq(self):
-        return ["\n\n", "Problem:"]
+        # return ["\n\n", "Problem:"]
+        return ["I hope it is correct."]
 
     def process_results(self, doc, results, params={}):
         candidates = results[0]