cleanup

docs/run-evals.md

@@ -10,7 +10,7 @@ oaieval gpt-3.5-turbo test-match
 ```
 The valid eval names are specified in the YAML files under `evals/registry/evals` and their corresponding implementations can be found in `evals/elsuite`.
 
-In this example, `gpt-3.5-turbo` is an OpenAI model that we dynamically instantiate as a completion function using `OpenAIChatCompletionFn(model=gpt-3.5-turbo)`. Any implementation of the `CompletionFn` protocol can be run against `oaieval`. By default, we support calling `oaieval` with any model availableon the OpenAI API or with CompletionFunctions available in [`evals/registry/completion_fns`](../evals/registry/completion_fns/). We are always interested in adding more completion functions and we encourage you to implement you own to reflect specific use cases.
+In this example, `gpt-3.5-turbo` is an OpenAI model that we dynamically instantiate as a completion function using `OpenAIChatCompletionFn(model=gpt-3.5-turbo)`. Any implementation of the `CompletionFn` protocol can be run against `oaieval`. By default, we support calling `oaieval` with any model available on the OpenAI API or with CompletionFunctions available in [`evals/registry/completion_fns`](../evals/registry/completion_fns/). We are always interested in adding more completion functions and we encourage you to implement you own to reflect specific use cases.
 
 More details on `CompletionFn` found here: [`completion-fns.md`](completion-fns.md)
 

evals/cli/oaieval.py

@@ -27,7 +27,7 @@ def get_parser() -> argparse.ArgumentParser:
  parser.add_argument(
  "completion_fn",
  type=str,
- help="One or more CompletionFn URLs, separated by commas (,). The format of a CompletionFn URL can be two forms: 1) an OpenAI API model followed by query parameters (e.g. `gpt-3.5-turbo?api_key=..`) or 2) a path to a Python class followed by query parameters (e.g. `evals:OpenAICompletionFn?model=text-davinci-003`).",
+ help="One or more CompletionFn URLs, separated by commas (,). A CompletionFn can either be the name of a model available in the OpenAI API or a key in the registry (see evals/registry/completion_fns).",
  )
  parser.add_argument("eval", type=str, help="Name of an eval. See registry.")
  parser.add_argument("--extra_eval_params", type=str, default="")

evals/completion_fns/openai.py

@@ -61,7 +61,7 @@ def __init__(
 
  def __call__(
  self,
- prompt: Union[OpenAICreatePrompt, Prompt],
+ prompt: Union[str, OpenAICreateChatPrompt],
  **kwargs,
  ) -> OpenAICompletionResult:
  if not isinstance(prompt, Prompt):
@@ -108,7 +108,7 @@ def __init__(
 
  def __call__(
  self,
- prompt: Union[OpenAICreateChatPrompt, Prompt],
+ prompt: Union[str, OpenAICreateChatPrompt],
  **kwargs,
  ) -> OpenAIChatCompletionResult:
  if not isinstance(prompt, Prompt):

evals/prompt/base.py

@@ -12,7 +12,7 @@
 ENCODER_LOCK = threading.Lock()
 
 # This is an approximation to the type accepted as the `prompt` field to `openai.Completion.create` calls
-OpenAICreatePrompt = Union[str, list[str]]
+OpenAICreatePrompt = Union[str, list[str], list[int], list[list[int]]]
 
 # This is the type accepted as the `prompt` field to `openai.ChatCompletion.create` calls
 OpenAIChatMessage = Dict[str, str] # A message is a dictionary with "role" and "content" keys