chain-of-thought for uni_finder and zhishu

evals/completion_fns/cot.py

@@ -5,6 +5,7 @@
 from evals.prompt.base import ChatCompletionPrompt
 from evals.record import record_sampling
 from evals.registry import Registry
+from evals.elsuite.utils import markdown_format_prompt
 
 DEFAULT_COT_TEMPLATE = "\nBefore answering, reason in a step-by-step manner as to get the right answer, then conclude with the answer."
 DEFAULT_EXTRACT_ANSWER_TEMPLATE = (
@@ -13,8 +14,9 @@
 
 
 class ChainOfThoughtCompletionResult(CompletionResult):
- def __init__(self, response) -> None:
+ def __init__(self, response, extras={}) -> None:
  self.response = response
+ self.extras = extras
 
  def get_completions(self) -> list[str]:
  return [self.response.strip()]
@@ -58,9 +60,13 @@ def __call__(self, prompt, **kwargs) -> ChainOfThoughtCompletionResult:
  {"role": "assistant", "content": answer},
  {"role": "assistant", "content": self.extract_answer_template},
  ]
+
+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("file")}
  extracted_answer = self.extract_completion_fn_instance(
  prompt=extraction_prompt, **kwargs
  ).get_completions()[0]
  record_sampling(prompt=extraction_prompt, sampled=extracted_answer)
 
- return ChainOfThoughtCompletionResult(extracted_answer)
+ return ChainOfThoughtCompletionResult(extracted_answer, {"cot_prompt": markdown_format_prompt(cot_prompt),
+ "extraction_prompt": markdown_format_prompt(extraction_prompt),
+ "answer": answer, "extracted_answer": extracted_answer})

evals/completion_fns/zhishu.py

@@ -45,9 +45,12 @@ class ZhishuCompletionResult(BaseCompletionResult):
  def get_completions(self) -> list[str]:
  completions = []
  if self.raw_data:
- for choice in self.raw_data.choices:
- if choice.message.content is not None:
- completions.append(choice.message.content)
+ if hasattr(self.raw_data, "choices"):
+ for choice in self.raw_data.choices:
+ if choice.message.content is not None:
+ completions.append(choice.message.content)
+ else:
+ completions.append("Error!")
  return completions
 
 

evals/elsuite/rag_match.py

@@ -104,11 +104,18 @@ def eval_sample(self, sample: Any, *_):
  )
  sampled = result.get_completions()[0]
 
+ extras = {}
+ if hasattr(result, "extras"):
+ if "extracted_answer" in result.extras:
+ sampled = result.extras["extracted_answer"].rstrip(".")
+ extras = result.extras
+
  return evals.record_and_check_match(
  prompt=prompt,
  sampled=sampled,
  expected=sample["ideal"],
- file_name=sample["file_name"]
+ file_name=sample["file_name"],
+ **extras
  )
 
  def run(self, recorder):

evals/elsuite/utils.py

@@ -192,3 +192,10 @@ def __call__(self, **kwargs):
  )
  sampled = result.get_completions()[0]
  return sampled, prompt
+
+
+def markdown_format_prompt(prompt):
+ if type(prompt) == list:
+ return "\n\n".join([f"**{message['role']}**: {message['content']}" for message in prompt])
+ else:
+ return prompt

evals/registry/completion_fns/uni_finder.yaml

@@ -1,4 +1,10 @@
 uni_finder-v1.26:
  class: evals.completion_fns.uni_finder:UniFinderCompletionFn
  args:
- pdf_parse_mode: v1.26
+ pdf_parse_mode: v1.26
+
+
+uni_finder-v1.26-cot:
+ class: evals.completion_fns.cot:ChainOfThoughtCompletionFn
+ args:
+ cot_completion_fn: uni_finder-v1.26

evals/registry/completion_fns/zhishu.yaml

@@ -9,3 +9,8 @@ zhishu/gpt-4-all:
  args:
  model: gpt-4-all
  all_tools: True
+
+zhishu/gpt-4-cot:
+ class: evals.completion_fns.cot:ChainOfThoughtCompletionFn
+ args:
+ cot_completion_fn: zhishu/gpt-4