get oaieval to run

evals/cli/oaieval.py

@@ -31,7 +31,6 @@ def get_parser() -> argparse.ArgumentParser:
  )
  parser.add_argument("eval", type=str, help="Name of an eval. See registry.")
  parser.add_argument("--extra_eval_params", type=str, default="")
- parser.add_argument("--modelspec_extra_options", type=str, default="")
  parser.add_argument("--max_samples", type=int, default=None)
  parser.add_argument("--cache", action=argparse.BooleanOptionalAction, default=True)
  parser.add_argument("--visible", action=argparse.BooleanOptionalAction, default=None)
@@ -110,6 +109,25 @@ def run(args, registry: Optional[Registry] = None):
  run_url = f"{run_spec.run_id}"
  logger.info(_purple(f"Run started: {run_url}"))
 
+ def parse_extra_eval_params(param_str: Optional[str]) -> Mapping[str, Any]:
+ """Parse a string of the form "key1=value1,key2=value2" into a dict."""
+ if not param_str:
+ return {}
+
+ def to_number(x):
+ try:
+ return int(x)
+ except:
+ pass
+ try:
+ return float(x)
+ except:
+ pass
+ return x
+
+ str_dict = dict(kv.split("=") for kv in param_str.split(","))
+ return {k: to_number(v) for k, v in str_dict.items()}
+
  extra_eval_params = parse_extra_eval_params(args.extra_eval_params)
 
  eval_class = registry.get_class(eval_spec)
@@ -143,7 +161,7 @@ def main():
  logging.getLogger("openai").setLevel(logging.WARN)
  if hasattr(openai.error, "set_display_cause"):
  openai.error.set_display_cause()
- run(args, model_resolver=ModelResolver())
+ run(args)
 
 
 if __name__ == "__main__":

evals/elsuite/modelgraded/classify.py

@@ -11,8 +11,6 @@
 import evals
 import evals.record
 from evals import CompletionFn, DummyCompletionFn, OpenAIChatCompletionFn
-from evals.elsuite.utils import PromptFn, format_necessary, scrub_formatting_from_prompt
-
 from evals.elsuite.modelgraded.base import ModelGradedSpec
 from evals.elsuite.modelgraded.classify_utils import (
  CHOICE_KEY,
@@ -151,7 +149,7 @@ def eval_sample(self, test_sample: dict, rng: Random) -> None:
  args = {k: v[1] for k, v in args.items()}
  prompt = self.mg.format(**args, **completions, **test_sample)
  evaluate = PromptFn(
- self.prompt,
+ prompt,
  completion_fn=self.eval_completion_fn,
  max_tokens=self.max_tokens,
  )

evals/registry.py

@@ -20,6 +20,7 @@
 from evals import OpenAIChatCompletionFn, OpenAICompletionFn
 from evals.api import CompletionFn, DummyCompletionFn
 from evals.base import BaseEvalSpec, CompletionFnSpec, EvalSetSpec, EvalSpec
+from evals.elsuite.modelgraded.base import ModelGradedSpec
 from evals.utils.misc import make_object
 
 logger = logging.getLogger(__name__)
@@ -147,10 +148,11 @@ def get_alias():
  except TypeError as e:
  raise TypeError(f"Error while processing {object} '{name}': {e}")
 
- def get_model(self, name: str) -> ModelSpec:
- return self._dereference(name, self._models, "model", ModelSpec)
-
  def get_modelgraded_spec(self, name: str, **kwargs: dict) -> dict[str, Any]:
+ assert name in self._modelgraded_specs, (
+ f"Modelgraded spec {name} not found. "
+ f"Closest matches: {difflib.get_close_matches(name, self._modelgraded_specs.keys(), n=5)}"
+ )
  return self._dereference(
  name, self._modelgraded_specs, "modelgraded spec", ModelGradedSpec, **kwargs
  )
@@ -266,9 +268,5 @@ def _evals(self):
  def _modelgraded_specs(self):
  return self._load_registry([p / "modelgraded" for p in self._registry_paths])
 
- @functools.cached_property
- def _models(self):
- return self._load_registry([p / "models" for p in self._registry_paths])
-
 
 registry = Registry()