From 6cc6a371fa94ab1354e2cf9f0cd4c95ccce0f3c6 Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 06:33:26 +0000 Subject: [PATCH 1/7] feat(dspy): in extend_generation, compare key values to None, not to --- dsp/primitives/predict.py | 88 ++++++++++++++++++++-------------- tests/datasets/test_dataset.py | 5 +- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.py index 41edf6c51..2921c6343 100644 --- a/dsp/primitives/predict.py +++ b/dsp/primitives/predict.py @@ -61,6 +61,41 @@ def _generate(template: Template, **kwargs) -> Callable: generator = dsp.settings.lm + def extend_generation(completion: Example, field_names: list[str], stage:str, max_depth: int, original_example:Example): + """If the required fields are not present in the completion, extend the generation.""" + assert max_depth > 0, "Max depth exceeded - failed to complete in one pass - increase max_tokens" + # remove content of last field to avoid half-completed content + for field_name in get_all_fields_following_missing_field(completion, field_names): + completion.pop(field_name, None) + + # Recurse with greedy decoding and a shorter length. + max_tokens = (kwargs.get("max_tokens") or + kwargs.get("max_output_tokens") or + dsp.settings.lm.kwargs.get("max_tokens") or + dsp.settings.lm.kwargs.get('max_output_tokens')) + + + if max_tokens is None: + raise ValueError("Required 'max_tokens' or 'max_output_tokens' not specified in settings.") + max_tokens = min(max(75, max_tokens // 2), max_tokens) + keys = list(kwargs.keys()) + list(dsp.settings.lm.kwargs.keys()) + max_tokens_key = "max_tokens" if "max_tokens" in keys else "max_output_tokens" + new_kwargs = { + **kwargs, + max_tokens_key: max_tokens, + "n": 1, + "temperature": 0.0, + } + + _, finished_completion = generate(template, **new_kwargs)( + completion, + stage=stage, + max_depth=max_depth - 1, + original_example=original_example, + ) + return finished_completion.data[0] + + def do_generate( example: Example, stage: str, max_depth: int = 2, original_example=None, ): @@ -77,45 +112,19 @@ def do_generate( completions: list[dict[str, Any]] = generator(prompt, **kwargs) completions: list[Example] = [template.extract(example, p) for p in completions] - # Find the completions that are most complete. + # Find the completions that are unfinished. field_names: list[str] = [field.input_variable for field in template.fields] - last_field_idx = 0 - for field_idx, key in enumerate(field_names): - completions_ = [ - c for c in completions if key in c.keys() and c[key] is not None - ] - - # Filter out completions that are missing fields that are present in at least one completion. - if len(completions_): - completions = completions_ - last_field_idx = field_idx + 1 - - # If none of the completions is completed (i.e., none has the final field set). - if last_field_idx < len(field_names): - # Pick the first completion that has gone farthest. - completion = completions[0] - completion[field_names[last_field_idx]] = "" - - # Recurse with greedy decoding and a shorter length. - max_tokens = kwargs.get("max_tokens", dsp.settings.lm.kwargs["max_tokens"]) - max_tokens = min(max(75, max_tokens // 2), max_tokens) - new_kwargs = { - **kwargs, - "max_tokens": max_tokens, - "n": 1, - "temperature": 0.0, - } - - assert max_depth > 0 - return generate(template, **new_kwargs)( - completion, - stage=stage, - max_depth=max_depth - 1, - original_example=original_example, + finished_completions = [] + for completion in completions: + if all((completion.get(key, None) is not None) for key in field_names): + finished_completions.append(completion) + continue + finished_completions.append( + extend_generation(completion, field_names, stage, max_depth, original_example), ) - completions = Completions(completions, template=template) + completions = Completions(finished_completions, template=template) example = example.copy(completions=completions) if len(completions) == 1: @@ -152,6 +161,15 @@ def do_generate( return do_generate +def get_all_fields_following_missing_field(completion: Example, field_names: list[str]) -> list[str]: + """Returns every field following the first missing field""" + for i, field_name in enumerate(field_names): + if field_name not in completion: + return field_names[i:] + if completion[field_name] == "": + return field_names[i:] + return [] + def generate_sc( example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs, diff --git a/tests/datasets/test_dataset.py b/tests/datasets/test_dataset.py index 4e0daf371..2c1e9af84 100644 --- a/tests/datasets/test_dataset.py +++ b/tests/datasets/test_dataset.py @@ -11,9 +11,6 @@ "This is content 2","What is that?","This is answer 2" """ -with open("dummy.csv", "w") as file: - file.write(dummy_data) - class CSVDataset(Dataset): def __init__(self, file_path, input_keys=None, *args, **kwargs) -> None: @@ -46,4 +43,6 @@ def test_input_keys(self): if __name__ == "__main__": + with open("dummy.csv", "w") as file: + file.write(dummy_data) unittest.main() From 2b55ca39b14390580b9b1e2b15539cbde12be182 Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 07:14:14 +0000 Subject: [PATCH 2/7] feat(dspy): add tests for extend generation --- tests/predict/test_predict.py | 89 +++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index c701b380a..48ecd2d52 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -1,11 +1,14 @@ -import dspy -from dspy import Predict, Signature, TypedPredictor -from dspy.utils.dummies import DummyLM import copy import textwrap + import pydantic +import pytest import ujson +import dspy +from dspy import Predict, Signature, TypedPredictor +from dspy.utils.dummies import DummyLM + def test_initialization_with_string_signature(): signature_string = "input1, input2 -> output" @@ -209,14 +212,84 @@ class OutputOnlySignature(dspy.Signature): assert lm.get_convo(-1) == textwrap.dedent( """\ Given the fields , produce the fields `output`. - + --- - + Follow the following format. - + Output: ${output} - + --- - + Output: short answer""" ) + + +@pytest.fixture(name="SandwichIdea") +def sandwich_idea_signature(): + class SandwichIdea(dspy.Signature): + """Based on the meal and dietary requirements, suggest a sandwich idea.""" + + meal: str = dspy.InputField() + dietary_requiements: str = dspy.InputField() + bread: str = dspy.OutputField() + protein: str = dspy.OutputField() + fat: str = dspy.OutputField() + garnish: str = dspy.OutputField() + sauce: str = dspy.OutputField() + + return SandwichIdea + + +def test_extend_generation(SandwichIdea): + # test that the completion is extended correctly + lm = DummyLM( + [ + " whole wheat\n\nProtein: turkey\n\nFat: avocado", + " tomato\n\nSauce: mustard", + ] + ) + dspy.settings.configure(lm=lm) + + prediction = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") + assert prediction.bread == "whole wheat" + assert prediction.protein == "turkey" + assert prediction.fat == "avocado" + assert prediction.garnish == "tomato" + assert prediction.sauce == "mustard" + + +def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): + # test the completion is rolled back when a field is skipped + lm = DummyLM( + [ + " white\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", + " ham\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", + ] + ) + dspy.settings.configure(lm=lm) + + predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") + assert predictor.bread == "white" + assert predictor.protein == "ham" + assert predictor.fat == "butter" + assert predictor.garnish == "lettuce" + assert predictor.sauce == "mayo" + + +def test_extend_generation_with_empty_field(SandwichIdea): + # test the completion is extended if the field is empty + lm = DummyLM( + [ + " white\n\nProtein: \n\nFat: butter\n\nGarish: lettuce", + " mayo", + ] + ) + dspy.settings.configure(lm=lm) + + predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") + assert predictor.bread == "white" + assert predictor.protein == "" + assert predictor.fat == "butter" + assert predictor.garnish == "lettuce" + assert predictor.sauce == "mayo" From d12e0f2b15da5f2a81fc4286d443c37eeb13ea9e Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 07:17:42 +0000 Subject: [PATCH 3/7] feat(dspy): revert changes to extend generation logic --- dsp/primitives/predict.py | 88 ++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.py index 2921c6343..41edf6c51 100644 --- a/dsp/primitives/predict.py +++ b/dsp/primitives/predict.py @@ -61,41 +61,6 @@ def _generate(template: Template, **kwargs) -> Callable: generator = dsp.settings.lm - def extend_generation(completion: Example, field_names: list[str], stage:str, max_depth: int, original_example:Example): - """If the required fields are not present in the completion, extend the generation.""" - assert max_depth > 0, "Max depth exceeded - failed to complete in one pass - increase max_tokens" - # remove content of last field to avoid half-completed content - for field_name in get_all_fields_following_missing_field(completion, field_names): - completion.pop(field_name, None) - - # Recurse with greedy decoding and a shorter length. - max_tokens = (kwargs.get("max_tokens") or - kwargs.get("max_output_tokens") or - dsp.settings.lm.kwargs.get("max_tokens") or - dsp.settings.lm.kwargs.get('max_output_tokens')) - - - if max_tokens is None: - raise ValueError("Required 'max_tokens' or 'max_output_tokens' not specified in settings.") - max_tokens = min(max(75, max_tokens // 2), max_tokens) - keys = list(kwargs.keys()) + list(dsp.settings.lm.kwargs.keys()) - max_tokens_key = "max_tokens" if "max_tokens" in keys else "max_output_tokens" - new_kwargs = { - **kwargs, - max_tokens_key: max_tokens, - "n": 1, - "temperature": 0.0, - } - - _, finished_completion = generate(template, **new_kwargs)( - completion, - stage=stage, - max_depth=max_depth - 1, - original_example=original_example, - ) - return finished_completion.data[0] - - def do_generate( example: Example, stage: str, max_depth: int = 2, original_example=None, ): @@ -112,19 +77,45 @@ def do_generate( completions: list[dict[str, Any]] = generator(prompt, **kwargs) completions: list[Example] = [template.extract(example, p) for p in completions] - # Find the completions that are unfinished. + # Find the completions that are most complete. field_names: list[str] = [field.input_variable for field in template.fields] - finished_completions = [] - for completion in completions: - if all((completion.get(key, None) is not None) for key in field_names): - finished_completions.append(completion) - continue - finished_completions.append( - extend_generation(completion, field_names, stage, max_depth, original_example), + last_field_idx = 0 + for field_idx, key in enumerate(field_names): + completions_ = [ + c for c in completions if key in c.keys() and c[key] is not None + ] + + # Filter out completions that are missing fields that are present in at least one completion. + if len(completions_): + completions = completions_ + last_field_idx = field_idx + 1 + + # If none of the completions is completed (i.e., none has the final field set). + if last_field_idx < len(field_names): + # Pick the first completion that has gone farthest. + completion = completions[0] + completion[field_names[last_field_idx]] = "" + + # Recurse with greedy decoding and a shorter length. + max_tokens = kwargs.get("max_tokens", dsp.settings.lm.kwargs["max_tokens"]) + max_tokens = min(max(75, max_tokens // 2), max_tokens) + new_kwargs = { + **kwargs, + "max_tokens": max_tokens, + "n": 1, + "temperature": 0.0, + } + + assert max_depth > 0 + return generate(template, **new_kwargs)( + completion, + stage=stage, + max_depth=max_depth - 1, + original_example=original_example, ) - completions = Completions(finished_completions, template=template) + completions = Completions(completions, template=template) example = example.copy(completions=completions) if len(completions) == 1: @@ -161,15 +152,6 @@ def do_generate( return do_generate -def get_all_fields_following_missing_field(completion: Example, field_names: list[str]) -> list[str]: - """Returns every field following the first missing field""" - for i, field_name in enumerate(field_names): - if field_name not in completion: - return field_names[i:] - if completion[field_name] == "": - return field_names[i:] - return [] - def generate_sc( example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs, From ea29442c7aea461e91e5b6d14edaafb1075e91f4 Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 07:18:15 +0000 Subject: [PATCH 4/7] feat(dspy): rm redundant comments --- tests/predict/test_predict.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 48ecd2d52..554b4dc28 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -242,7 +242,6 @@ class SandwichIdea(dspy.Signature): def test_extend_generation(SandwichIdea): - # test that the completion is extended correctly lm = DummyLM( [ " whole wheat\n\nProtein: turkey\n\nFat: avocado", @@ -260,7 +259,6 @@ def test_extend_generation(SandwichIdea): def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): - # test the completion is rolled back when a field is skipped lm = DummyLM( [ " white\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", @@ -278,7 +276,6 @@ def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): def test_extend_generation_with_empty_field(SandwichIdea): - # test the completion is extended if the field is empty lm = DummyLM( [ " white\n\nProtein: \n\nFat: butter\n\nGarish: lettuce", From 15aef81a2182568795b29ae3dffd60f7251c3c0e Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 07:28:13 +0000 Subject: [PATCH 5/7] feat(dspy): revert changes to test file writing --- tests/datasets/test_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/datasets/test_dataset.py b/tests/datasets/test_dataset.py index 2c1e9af84..4e0daf371 100644 --- a/tests/datasets/test_dataset.py +++ b/tests/datasets/test_dataset.py @@ -11,6 +11,9 @@ "This is content 2","What is that?","This is answer 2" """ +with open("dummy.csv", "w") as file: + file.write(dummy_data) + class CSVDataset(Dataset): def __init__(self, file_path, input_keys=None, *args, **kwargs) -> None: @@ -43,6 +46,4 @@ def test_input_keys(self): if __name__ == "__main__": - with open("dummy.csv", "w") as file: - file.write(dummy_data) unittest.main() From 477153561ebe4780b3922d39018b13fd8d6a02d0 Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Thu, 20 Jun 2024 07:20:55 +0000 Subject: [PATCH 6/7] feat(dspy): changed assertions so tests pass and added conversation logs --- tests/predict/test_predict.py | 245 ++++++++++++++++++++++++++++++++-- 1 file changed, 233 insertions(+), 12 deletions(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 554b4dc28..0e8b67790 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -245,32 +245,182 @@ def test_extend_generation(SandwichIdea): lm = DummyLM( [ " whole wheat\n\nProtein: turkey\n\nFat: avocado", - " tomato\n\nSauce: mustard", + # Incomplete generation leads to tomato field being assigned as an + # empty string ("") in dsp.primitives.predict l98 the generation + # therefores continues with the next field. + " tomato \n\nSauce: mustard", ] ) dspy.settings.configure(lm=lm) prediction = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") + + # The logged conversation: + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: whole wheat + + # Protein: turkey + + # Fat: avocado + # === + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: whole wheat + + # Protein: turkey + + # Fat: avocado + + # Garnish: tomato + + # Sauce: mustard + # === + assert prediction.bread == "whole wheat" assert prediction.protein == "turkey" assert prediction.fat == "avocado" - assert prediction.garnish == "tomato" - assert prediction.sauce == "mustard" + assert prediction.garnish == "" # This field is assigned as "" when the generation is extended + assert prediction.sauce == "tomato \n\nSauce: mustard" def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): lm = DummyLM( [ - " white\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", - " ham\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", + " white\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo", + " ham\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo", ] ) dspy.settings.configure(lm=lm) + # The logged conversation: + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: white + + # Fat: butter + + # Garnish: lettuce + + # Sauce: mayo + # === + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: white Fat: butter Garnish: lettuce Sauce: mayo + + # Protein: ham + + # Fat: butter + + # Garnish: lettuce + + # Sauce: mayo + # === + predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") - assert predictor.bread == "white" - assert predictor.protein == "ham" - assert predictor.fat == "butter" + assert predictor.bread == "white\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo" + assert predictor.protein == "" + assert predictor.fat == "ham\n\nFat: butter" assert predictor.garnish == "lettuce" assert predictor.sauce == "mayo" @@ -278,15 +428,86 @@ def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): def test_extend_generation_with_empty_field(SandwichIdea): lm = DummyLM( [ - " white\n\nProtein: \n\nFat: butter\n\nGarish: lettuce", - " mayo", + " white\n\nProtein: \n\nFat: butter\n\nGarnish: lettuce", + " lettuce \n\nSauce: mayo", ] ) dspy.settings.configure(lm=lm) + # The logged conversation: + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: white + + # Protein: + + # Fat: butter + + # Garnish: lettuce + # === + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: white + + # Protein: Fat: butter Garnish: lettuce + + # Fat: lettuce + + # Sauce: mayo + # === predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") assert predictor.bread == "white" - assert predictor.protein == "" - assert predictor.fat == "butter" + assert predictor.protein == "Fat: butter\n\nGarnish: lettuce" + assert predictor.fat == "" assert predictor.garnish == "lettuce" assert predictor.sauce == "mayo" From bcab220171d5f02fbc2833bce649f296519cf3bb Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Thu, 20 Jun 2024 07:35:25 +0000 Subject: [PATCH 7/7] feat(dsp): clarify what is generated in each generation --- tests/predict/test_predict.py | 129 ++++------------------------------ 1 file changed, 15 insertions(+), 114 deletions(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 0e8b67790..bd07e776e 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -254,77 +254,44 @@ def test_extend_generation(SandwichIdea): dspy.settings.configure(lm=lm) prediction = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") - - # The logged conversation: + # The logged conversation (additional newlines removed, [..] indicates the generation): # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - - # Bread: whole wheat - + # Bread: [whole wheat # Protein: turkey - - # Fat: avocado + # Fat: avocado] # === # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - # Bread: whole wheat - # Protein: turkey - # Fat: avocado - - # Garnish: tomato - - # Sauce: mustard + # Garnish: [tomato + # Sauce: mustard] # === assert prediction.bread == "whole wheat" @@ -342,79 +309,45 @@ def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): ] ) dspy.settings.configure(lm=lm) - - # The logged conversation: + # The logged conversation (additional newlines removed, [..] indicates the generation): # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - - # Bread: white - + # Bread:[ white # Fat: butter - # Garnish: lettuce - - # Sauce: mayo + # Sauce: mayo] # === # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - # Bread: white Fat: butter Garnish: lettuce Sauce: mayo - - # Protein: ham - + # Protein:[ ham # Fat: butter - # Garnish: lettuce - - # Sauce: mayo + # Sauce: mayo] # === predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") @@ -433,76 +366,44 @@ def test_extend_generation_with_empty_field(SandwichIdea): ] ) dspy.settings.configure(lm=lm) - # The logged conversation: + # The logged conversation (additional newlines removed, [..] indicates the generation): # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - - # Bread: white - + # Bread:[ white # Protein: - # Fat: butter - - # Garnish: lettuce + # Garnish: lettuce] # === # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - # Bread: white - # Protein: Fat: butter Garnish: lettuce - - # Fat: lettuce - - # Sauce: mayo + # Fat:[ lettuce + # Sauce: mayo] # === predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A")