From 30b4630877c5dc8177a9bcdea7004c94bcb13f6f Mon Sep 17 00:00:00 2001
From: Andrew Kondrich <120423412+andrew-openai@users.noreply.github.com>
Date: Fri, 14 Apr 2023 16:32:37 -0700
Subject: [PATCH] [Evals] support kwargs in eval all samples (#680)

---
 evals/eval.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/evals/eval.py b/evals/eval.py
index 7e9a1775be..b914565096 100644
--- a/evals/eval.py
+++ b/evals/eval.py
@@ -14,9 +14,9 @@
 
 from evals.api import CompletionFn
 
+from .data import get_jsonl
 from .record import RecorderBase
 from .registry import Registry
-from .data import get_jsonl
 
 logger = logging.getLogger(__name__)
 
@@ -88,6 +88,7 @@ async def async_eval_all_samples(
         samples: List[Any],
         concurrency: int = 32,
         show_progress: bool = True,
+        **_kwargs: Any,
     ):
         work_items = _index_samples(samples)
         semaphore = asyncio.Semaphore(concurrency)
@@ -109,6 +110,7 @@ def eval_all_samples(
         samples,
         show_progress=True,
         record_raw_sample=True,
+        **_kwargs: Any,
     ):
         """
         Evaluate all provided samples in parallel.
@@ -156,7 +158,7 @@ def worker_thread(args):
     def get_samples(self):
         if self.samples_jsonl is None:
             raise ValueError(
-                    "To use `get_samples`, you must provide a `samples_jsonl` path."
-                    "Got `None`.")
+                "To use `get_samples`, you must provide a `samples_jsonl` path." "Got `None`."
+            )
 
         return get_jsonl(self.samples_jsonl)