forked from openai/evals
-
Notifications
You must be signed in to change notification settings - Fork 0
/
pattern_identification_generator.py
70 lines (58 loc) · 2.67 KB
/
pattern_identification_generator.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
"""
Usage: python scripts/task_identification_generator.py
This script generates examples for a task identification challenge,
with the task being to identify patterns between a set of symbols and their resulting labels.
"""
import json
import os
import random
from typing import Literal
# Ensure consistent results across runs
random.seed(42)
SYMBOLS = list("abcdefghijklmnopqrstuvwxyz")
DELIMETER = "->"
INSTRUCTION = (
'Figure out the pattern in the below examples, and then answer with just "foo" or "bar".'
)
TASK_NAME = "pattern_identification"
# This function generates an example symbol set and its corresponding label
def generate_example() -> tuple[str, list[str], Literal["foo", "bar"]]:
num_symbols = int(len(SYMBOLS) / 2)
target_symbol = random.choice(SYMBOLS)
symbol_list = random.sample(SYMBOLS, num_symbols)
target: Literal["foo", "bar"] = "foo" if target_symbol in symbol_list else "bar"
return (target_symbol, symbol_list, target)
# This function generates a string of multiple examples, used to give a user multiple attempts to identify the pattern
def generate_exemplars_str(num_exemplars: int = 8) -> str:
exemplars = [generate_example() for _ in range(num_exemplars)]
exemplars_str = [
f"({exemplar[0]}, {exemplar[1]}) {DELIMETER} {exemplar[2]}".replace("'", "")
for exemplar in exemplars
]
return "\n".join([INSTRUCTION] + exemplars_str)
# This function generates a set of evaluation examples and their corresponding labels
def generate_eval_examples(
num_eval_examples: int = 250,
) -> tuple[list[str], list[Literal["foo", "bar"]]]:
eval_examples = [generate_example() for _ in range(num_eval_examples)]
eval_examples_str = [
f"{generate_exemplars_str()}\n({example[0]}, {example[1]}) {DELIMETER}".replace("'", "")
for example in eval_examples
]
targets: list[Literal["foo", "bar"]] = [example[2] for example in eval_examples]
return eval_examples_str, targets
if __name__ == "__main__":
eval_examples_str, targets = generate_eval_examples()
# Generate the output path in a OS-agnostic manner
output_path = os.path.join("evals", "registry", "data", TASK_NAME, "samples.v0.jsonl")
with open(output_path, "w") as writer:
for eval_example_str, target in zip(eval_examples_str, targets):
d = {
"input": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": eval_example_str},
],
"ideal": target,
}
writer.write(json.dumps(d) + "\n")
print(f"{len(eval_examples_str)} lines written to {output_path}.")