From 2486f7e3a3247ccc388c37f7459736af13a2dca1 Mon Sep 17 00:00:00 2001 From: Shane Gu <343165+rlbayes@users.noreply.github.com> Date: Wed, 29 Mar 2023 20:25:30 -0700 Subject: [PATCH] [evals] added eval_model flag to modelgraded eval (#519) --- evals/elsuite/modelgraded/classify.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/evals/elsuite/modelgraded/classify.py b/evals/elsuite/modelgraded/classify.py index 3e6d29fd71..fc6911d270 100644 --- a/evals/elsuite/modelgraded/classify.py +++ b/evals/elsuite/modelgraded/classify.py @@ -100,6 +100,7 @@ def __init__( multicomp_temperature: float = 0.4, samples_renamings: Optional[dict[str, str]] = None, eval_type: Optional[str] = None, + eval_model: str = "gpt-3.5-turbo", metaeval: bool = False, modelgraded_spec_args: Optional[dict[str, dict[str, str]]] = None, **kwargs, @@ -130,9 +131,7 @@ def __init__( if self.model_spec.name == "dummy-completion" or self.model_spec.name == "dummy-chat": self.eval_modelspec = self.model_spec else: - self.eval_modelspec = ModelSpec( - name="gpt-3.5-turbo", model="gpt-3.5-turbo", is_chat=True - ) + self.eval_modelspec = ModelSpec(name=eval_model, model=eval_model, is_chat=True) """import prompt and set attributes""" modelgraded_specs = self.registry.get_modelgraded_spec(modelgraded_spec)