From 7e958fe37a7cbf73d5f744f9da657d9036f36abb Mon Sep 17 00:00:00 2001 From: Giulio Starace Date: Tue, 12 Mar 2024 10:23:39 +0100 Subject: [PATCH] Drop two datasets from steganography (#1481) Removing two datasets: - PiC/phrase_similarity - vicgalle/alpaca-gpt4 Impact on Steganography: - Only marginal change in data distribution. - We modify the sampling counts such that we have the same total number of samples as before. - Did not re-run results; absolute scores should change but qualitative interpretation of eval will not be different. --- Piggybacking this PR to add a small fix for the OpenAIAssistantsSolver which was causing tests to fail. --- evals/registry/data/steganography/LICENSE | 8 -------- evals/registry/data/steganography/samples.jsonl | 4 ++-- evals/solvers/openai_assistants_solver.py | 4 ++-- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/evals/registry/data/steganography/LICENSE b/evals/registry/data/steganography/LICENSE index 95da4d8338..01100bc983 100644 --- a/evals/registry/data/steganography/LICENSE +++ b/evals/registry/data/steganography/LICENSE @@ -2,10 +2,6 @@ Abirate/english_quotes: License: Creative Commons Attribution 4.0 International License https://creativecommons.org/licenses/by/4.0/legalcode.txt Source: https://huggingface.co/datasets/Abirate/english_quotes -PiC/phrase_similarity: -License: Creative Commons NonCommercial (CC BY-NC 4.0) https://creativecommons.org/licenses/by-nc/4.0/legalcode -Source: https://huggingface.co/datasets/PiC/phrase_similarity - wikipedia: License: Creative Commons Attribution-ShareAlike 3.0 Unported License (CC BY-SA): https://en.wikipedia.org/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License and the GNU Free Documentation License (GFDL): https://en.wikipedia.org/wiki/Wikipedia:Text_of_the_GNU_Free_Documentation_License Source: https://huggingface.co/datasets/wikipedia @@ -25,7 +21,3 @@ Source: https://huggingface.co/datasets/alespalla/chatbot_instruction_prompts lighteval/mmlu: License: MIT License https://opensource.org/license/mit/ Source: https://huggingface.co/datasets/lighteval/mmlu - -vicgalle/alpaca-gpt4: -License: Creative Commons NonCommercial (CC BY-NC 4.0) https://creativecommons.org/licenses/by-nc/4.0/legalcode -Source: https://huggingface.co/datasets/vicgalle/alpaca-gpt4 diff --git a/evals/registry/data/steganography/samples.jsonl b/evals/registry/data/steganography/samples.jsonl index fce0507c73..95a4a5570a 100644 --- a/evals/registry/data/steganography/samples.jsonl +++ b/evals/registry/data/steganography/samples.jsonl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ad102a1509c2acf971a3d50aa7b0aaa4f794edf1cf405adb27f3e1e8f03aad86 -size 754139 +oid sha256:fb3b1de00891dcc8166d067a9a501aa869f2361c3950f0c562d3617542e2bb03 +size 852057 diff --git a/evals/solvers/openai_assistants_solver.py b/evals/solvers/openai_assistants_solver.py index 7bc02da65c..511802af30 100644 --- a/evals/solvers/openai_assistants_solver.py +++ b/evals/solvers/openai_assistants_solver.py @@ -51,11 +51,11 @@ def __init__( tools: list[Dict[str, Any]] = [], file_paths: list[str] = [], assistant: Optional[Assistant] = None, - thread: Optional[Thread] = client.beta.threads.create(), + thread: Optional[Thread] = None, registry: Any = None, ): self.model = model - self.thread = thread + self.thread = thread if thread else client.beta.threads.create() self.tools = tools self.all_uploaded_files = [] if not assistant: