From acd77a08cd910e79f81c416c43e3c55fcdcdcfe9 Mon Sep 17 00:00:00 2001 From: djstrong Date: Fri, 3 May 2024 22:08:57 +0200 Subject: [PATCH 1/2] remove echo parameter in OpenAI completions API --- lm_eval/models/openai_completions.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lm_eval/models/openai_completions.py b/lm_eval/models/openai_completions.py index 87e4bd2bbe..c7c415d5ec 100644 --- a/lm_eval/models/openai_completions.py +++ b/lm_eval/models/openai_completions.py @@ -14,7 +14,7 @@ from lm_eval.utils import eval_logger -def get_result(response, ctxlen: int) -> Tuple[float, bool]: +def get_result(response) -> Tuple[float, bool]: """Process results from OpenAI API response. :param response: dict @@ -29,9 +29,9 @@ def get_result(response, ctxlen: int) -> Tuple[float, bool]: """ is_greedy = True logprobs = response.logprobs.token_logprobs - continuation_logprobs = sum(logprobs[ctxlen:]) + continuation_logprobs = sum(logprobs) - for i in range(ctxlen, len(response.logprobs.token_logprobs)): + for i in range(len(response.logprobs.token_logprobs)): token = response.logprobs.token_logprobs[i] top_tokens = response.logprobs.top_logprobs[i] top_token = max(top_tokens.keys(), key=lambda x: top_tokens[x]) @@ -212,7 +212,6 @@ def _collate(x): client=self.client, model=self.model, prompt=inps, - echo=True, max_tokens=0, temperature=0.0, logprobs=10, @@ -222,7 +221,7 @@ def _collate(x): for resp, ctxlen, (cache_key, context_enc, continuation_enc) in zip( response.choices, ctxlens, chunk ): - answer = get_result(resp, ctxlen) + answer = get_result(resp) res.append(answer) From a64b2450ca6a79774446af86560faeaa801bd2ca Mon Sep 17 00:00:00 2001 From: "kwrobel.eth" Date: Fri, 3 May 2024 22:26:47 +0200 Subject: [PATCH 2/2] remove context length parameter doc string --- lm_eval/models/openai_completions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lm_eval/models/openai_completions.py b/lm_eval/models/openai_completions.py index c7c415d5ec..c8f27ff75d 100644 --- a/lm_eval/models/openai_completions.py +++ b/lm_eval/models/openai_completions.py @@ -19,8 +19,6 @@ def get_result(response) -> Tuple[float, bool]: :param response: dict OpenAI API Response - :param ctxlen: int - Length of context (so we can slice them away and only keep the predictions) :return: continuation_logprobs: np.array Log probabilities of continuation tokens