From 9cf345255e1152fffbb8606ab91421ddfae3404c Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Tue, 7 May 2024 06:34:08 -0700 Subject: [PATCH 01/16] define ast abstraction layer --- .../python_protocol_generation/ast_helpers.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py b/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py new file mode 100644 index 00000000000..bba67650563 --- /dev/null +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py @@ -0,0 +1,104 @@ +"""Abstract layer for generating AST nodes. + +Provide primitive data structures that can be used to generate AST nodes. +""" + +import typing +import ast +from dataclasses import dataclass + + +class CanGenerateAST(typing.Protocol): + """Protocol for objects that can generate an AST node.""" + + def generate_ast(self) -> ast.AST: + """Generate an AST node.""" + ... + + +@dataclass +class ImportStatement: + """Class to represent from some.module import a_thing statement.""" + + module: str + names: typing.List[str] + + def generate_ast(self) -> ast.ImportFrom: + """Generate an AST node for the import statement.""" + return ast.ImportFrom( + module=self.module, + names=[ast.alias(name=name, asname=None) for name in self.names], + level=0, + ) + + +@dataclass +class CallFunction: + """Class to represent a method or function call.""" + + call_on: str + method_name: str + args: typing.List[str] + + def generate_ast(self) -> ast.Call: + """Generate an AST node for the call.""" + return ast.Call( + func=ast.Attribute( + value=ast.Name(id=self.call_on, ctx=ast.Load()), + attr=self.method_name, + ctx=ast.Load(), + ), + args=[ast.Constant(str_arg) for str_arg in self.args], + keywords=[], + ) + + +@dataclass +class AssignStatement: + """Class to represent an assignment statement.""" + + var_name: str + value: CallFunction | str | ast.AST + + def generate_ast(self) -> ast.Assign: + """Generate an AST node for the assignment statement.""" + if isinstance(self.value, CallFunction): + return ast.Assign( + targets=[ast.Name(id=self.var_name, ctx=ast.Store())], + value=self.value.generate_ast(), + ) + else: + return ast.Assign( + targets=[ast.Name(id=self.var_name, ctx=ast.Store())], + value=self.value, + ) + + +@dataclass +class FunctionDefinition: + """Class to represent a function definition.""" + + name: str + args: typing.List[str] + + def generate_ast(self) -> ast.FunctionDef: + """Generate an AST node for the function definition.""" + return ast.FunctionDef( + name=self.name, + args=ast.arguments( + posonlyargs=[], + args=[ + ast.arg( + arg=arg, + ) + for arg in self.args + ], + vararg=None, + kwonlyargs=[], + kw_defaults=[], + kwarg=None, + defaults=[], + ), + body=[], + decorator_list=[], + ) From 4d1fc3a5ffebd9b5ee8f84451a265780f15aa4b3 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Tue, 7 May 2024 06:35:18 -0700 Subject: [PATCH 02/16] define conversion layer from PossibleSlotContents to ast helper classes --- .../generation_phases/load_phase.py | 232 ++++++++++++++++++ .../generation_phases/setup_phase.py | 34 +++ 2 files changed, 266 insertions(+) create mode 100644 test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py create mode 100644 test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/setup_phase.py diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py new file mode 100644 index 00000000000..b78495658eb --- /dev/null +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py @@ -0,0 +1,232 @@ +"""This module contains the functions that generate the various load statements of a protocol. + +For example, load_module, load_labware, load_waste_chute, etc. +""" + +import typing +from test_data_generation.deck_configuration.datashapes import ( + PossibleSlotContents as PSC, + Slot, + SlotName, + RowName, +) +from test_data_generation.python_protocol_generation import ast_helpers as ast_h + + +def _staging_area(row: RowName) -> ast_h.AssignStatement: + """Create a staging area in a specified row. + + This is done implicitly by loading a 96-well plate in column 4 of the specified row. + """ + labware_name = "nest_96_wellplate_100ul_pcr_full_skirt" + labware_location = f"{row.upper()}4" + + return ast_h.AssignStatement( + var_name=f"well_plate_{row}4", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_labware", + args=[labware_name, labware_location], + ), + ) + + +def _waste_chute( + has_staging_area: bool, has_cover: bool +) -> typing.List[ast_h.AssignStatement]: + """Create a waste chute. + + If has_staging_area is True, a staging area is created in row D. + """ + entries = [ + ast_h.AssignStatement( + var_name="waste_chute", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_waste_chute", + args=[], + ), + ) + ] + + # TODO: If has_cover, set USE_96_CHANNEL_PIPETTE to False + + if has_staging_area: + entries.append(_staging_area("d")) + + return entries + + +def _magnetic_block_on_staging_area(row: RowName) -> typing.List[ast_h.AssignStatement]: + """Create a magnetic block on a staging area in a specified row.""" + module_name = "magneticBlockV1" + module_location = f"{row.upper()}3" + + entries = [ + ast_h.AssignStatement( + var_name=f"mag_block_{row}3", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_module", + args=[module_name, module_location], + ), + ), + _staging_area(row), + ] + return entries + + # Call module.labware to make sure it is included as part of the analysis + + +def _trash_bin(slot: SlotName) -> ast_h.AssignStatement: + """Create a trash bin in a specified slot.""" + location = slot.upper() + + return ast_h.AssignStatement( + var_name=f"trash_bin_{slot}", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_trash_bin", + args=[location], + ), + ) + + # Call trash_bin.top() to make sure it is included as part of the analysis + + +def _thermocycler_module() -> ast_h.AssignStatement: + """Create a thermocycler module.""" + module_name = "thermocyclerModuleV2" + + return ast_h.AssignStatement( + var_name="thermocycler_module", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_module", + args=[module_name], + ), + ) + + # Call module.labware to make sure it is included as part of the analysis + + +def _temperature_module(slot: SlotName) -> ast_h.AssignStatement: + """Create a temperature module in a specified slot.""" + module_name = "temperatureModuleV2" + module_location = slot.upper() + return ast_h.AssignStatement( + var_name=f"temperature_module_{slot}", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_module", + args=[module_name, module_location], + ), + ) + + # Call module.labware to make sure it is included as part of the analysis + + +def _magnetic_block(slot_name: SlotName) -> ast_h.AssignStatement: + """Create a magnetic block in a specified slot.""" + module_name = "magneticBlockV1" + module_location = slot_name.upper() + return ast_h.AssignStatement( + var_name=f"mag_block_{slot_name}", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_module", + args=[module_name, module_location], + ), + ) + # Call module.labware to make sure it is included as part of the analysis + + +def _heater_shaker_module(slot_name: SlotName) -> ast_h.AssignStatement: + """Create a heater shaker module in a specified slot.""" + module_name = "heaterShakerModuleV1" + module_location = slot_name.upper() + + return ast_h.AssignStatement( + var_name=f"heater_shaker_{slot_name}", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_module", + args=[module_name, module_location], + ), + ) + # Call module.labware to make sure it is included as part of the analysis + + +def _labware_slot(slot_name: SlotName) -> ast_h.AssignStatement: + """Create a labware slot in a specified slot.""" + labware_name = "nest_96_wellplate_100ul_pcr_full_skirt" + labware_location = slot_name.upper() + + return ast_h.AssignStatement( + var_name=f"well_plate_{slot_name}", + value=ast_h.CallFunction( + call_on="protocol_context", + method_name="load_labware", + args=[labware_name, labware_location], + ), + ) + # well_plate_{slot}.is_tiprack + + +def create_load_statement( + slot: Slot, +) -> ast_h.AssignStatement | typing.List[ast_h.AssignStatement]: + """Maps the contents of a slot to the correct assign statement.""" + match slot.contents: + case PSC.WASTE_CHUTE: + return _waste_chute(False, True) + + case PSC.WASTE_CHUTE_NO_COVER: + return _waste_chute(False, False) + + case PSC.STAGING_AREA_WITH_WASTE_CHUTE: + return _waste_chute(True, True) + + case PSC.STAGING_AREA_WITH_WASTE_CHUTE_NO_COVER: + return _waste_chute(True, False) + + case PSC.STAGING_AREA_WITH_MAGNETIC_BLOCK: + return _magnetic_block_on_staging_area(slot.row) + + case PSC.TRASH_BIN: + return _trash_bin(slot.label) + + case PSC.THERMOCYCLER_MODULE: + return _thermocycler_module() + + case PSC.TEMPERATURE_MODULE: + return _temperature_module(slot.label) + + case PSC.MAGNETIC_BLOCK_MODULE: + return _magnetic_block(slot.label) + + case PSC.HEATER_SHAKER_MODULE: + return _heater_shaker_module(slot.label) + + case PSC.STAGING_AREA: + return _staging_area(slot.row) + + case PSC.LABWARE_SLOT: + return _labware_slot(slot.label) + + case _: + raise (ValueError(f"Unknown slot contents: {slot.contents}")) + + +def create_load_statements( + slots: typing.List[Slot], +) -> typing.List[ast_h.AssignStatement]: + """Iterates over a list of slots and creates the corresponding load statements.""" + entries: typing.List[ast_h.AssignStatement] = [] + for slot in slots: + load_statement = create_load_statement(slot) + if isinstance(load_statement, typing.List): + entries.extend(load_statement) + else: + entries.append(load_statement) + return entries diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/setup_phase.py b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/setup_phase.py new file mode 100644 index 00000000000..f065cbe302c --- /dev/null +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/setup_phase.py @@ -0,0 +1,34 @@ +"""This module provides function to generate the initial setup of an Opentrons protocol.""" +import ast +import typing +from test_data_generation.python_protocol_generation import ast_helpers as ast_h + + +def create_requirements_dict( + robot_type: typing.Literal["OT-2", "OT-3"], api_version: str +) -> ast_h.AssignStatement: + """Create an assignment statement for the requirements dictionary.""" + return ast_h.AssignStatement( + var_name="requirements", + value=ast.Expression( + body=ast.Dict( + keys=[ast.Constant("robotType"), ast.Constant("apiLevel")], + values=[ast.Constant(robot_type), ast.Constant(api_version)], + ), + ), + ) + + +def import_protocol_context() -> ast_h.ImportStatement: + """Create an import statement for the ProtocolContext class.""" + return ast_h.ImportStatement( + module="opentrons.protocol_api", names=["ProtocolContext"] + ) + + +def create_protocol_context_run_function() -> ast_h.FunctionDefinition: + """Create a function definition for the run function of a protocol.""" + return ast_h.FunctionDefinition( + name="run", + args=["protocol_context"], + ) From 9fd24ec17649b3d8d680bd9e1e56bb2e2d699a97 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Tue, 7 May 2024 06:36:19 -0700 Subject: [PATCH 03/16] generate Python code --- .../python_protocol_generation/__init__.py | 1 + .../python_protocol_generator.py | 49 +++++++++++++++++++ .../test_deck_configuration.py | 43 +++++++--------- 3 files changed, 69 insertions(+), 24 deletions(-) create mode 100644 test-data-generation/src/test_data_generation/python_protocol_generation/__init__.py create mode 100644 test-data-generation/src/test_data_generation/python_protocol_generation/python_protocol_generator.py diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/__init__.py b/test-data-generation/src/test_data_generation/python_protocol_generation/__init__.py new file mode 100644 index 00000000000..45f2dcce037 --- /dev/null +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/__init__.py @@ -0,0 +1 @@ +"""Test data generation.""" diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/python_protocol_generator.py b/test-data-generation/src/test_data_generation/python_protocol_generation/python_protocol_generator.py new file mode 100644 index 00000000000..113f1ccb9f2 --- /dev/null +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/python_protocol_generator.py @@ -0,0 +1,49 @@ +"""Module for generating Python protocol code from a deck configuration.""" + +import ast +import astor # type: ignore +import typing +from .ast_helpers import CanGenerateAST +from test_data_generation.deck_configuration.datashapes import DeckConfiguration +from .generation_phases.load_phase import create_load_statements +from .generation_phases.setup_phase import ( + create_protocol_context_run_function, + import_protocol_context, + create_requirements_dict, +) + + +class PythonProtocolGenerator: + """Class for generating Python protocol code from a deck configuration.""" + + def __init__( + self, + deck_configuration: DeckConfiguration, + api_version: str, + ) -> None: + """Initialize the PythonProtocolGenerator. + + Call boilerplate functions to set up the protocol. + """ + self._top_level_statements: typing.List[CanGenerateAST] = [] + self._deck_configuration = deck_configuration + + self._top_level_statements.extend( + [ + import_protocol_context(), + create_requirements_dict("OT-3", api_version), + ] + ) + + def generate_protocol(self) -> str: + """Generate the Python protocol code.""" + module = ast.Module( + body=[statement.generate_ast() for statement in self._top_level_statements] + ) + run_function = create_protocol_context_run_function().generate_ast() + load_entries = create_load_statements(self._deck_configuration.slots) + for entry in load_entries: + run_function.body.append(entry.generate_ast()) + module.body.append(run_function) + + return str(astor.to_source(module)) diff --git a/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py b/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py index 02c4f125187..07e189f0925 100644 --- a/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py +++ b/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py @@ -1,41 +1,36 @@ """Tests to ensure that the deck configuration is generated correctly.""" +import pytest +from pathlib import Path from hypothesis import given, settings, HealthCheck from test_data_generation.deck_configuration.datashapes import DeckConfiguration -from test_data_generation.deck_configuration.strategy.final_strategies import ( +from test_data_generation.deck_configuration.strategy.deck_configuration_strategies import ( a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker, a_deck_configuration_with_invalid_fixture_in_col_2, ) - -NUM_EXAMPLES = 100 +from test_data_generation.python_protocol_generation.python_protocol_generator import ( + PythonProtocolGenerator, +) @given( deck_config=a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker() ) -@settings( - max_examples=NUM_EXAMPLES, - suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], -) -def test_above_below_heater_shaker(deck_config: DeckConfiguration) -> None: +@settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) +@pytest.mark.asyncio +async def test_above_below_heater_shaker( + deck_config: DeckConfiguration, tmp_path: Path +) -> None: """I hypothesize, that any deck configuration with a non-labware slot fixture above or below a heater-shaker is invalid.""" - print(deck_config) - - # TODO: create protocol and run analysis - - # protocol = create_protocol(deck) - # with pytest.assertRaises as e: - # analyze(protocol) - # assert e.exception == "Some statement about the deck configuration being invalid because of the labware above or below the Heater-Shaker" + protocol_content = PythonProtocolGenerator(deck_config, "2.18").generate_protocol() + print(protocol_content) @given(deck_config=a_deck_configuration_with_invalid_fixture_in_col_2()) -@settings( - max_examples=NUM_EXAMPLES, - suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], -) -def test_invalid_fixture_in_col_2(deck_config: DeckConfiguration) -> None: +@settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) +def test_invalid_fixture_in_col_2( + deck_config: DeckConfiguration, tmp_path: Path +) -> None: """I hypothesize, that any deck configuration that contains at least one, Heater-Shaker, Trash Bin, or Temperature module, in column 2 is invalid.""" - print(deck_config) - - # TODO: Same as above + protocol_content = PythonProtocolGenerator(deck_config, "2.18").generate_protocol() + print(protocol_content) From 8c8ad3ed35b9929917eb712c7ac21d3fc110068c Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Tue, 7 May 2024 06:36:36 -0700 Subject: [PATCH 04/16] some cleanup stuff --- .../deck_configuration/datashapes.py | 4 ++-- ...es.py => deck_configuration_strategies.py} | 9 ++++++++ test-data-generation/tests/conftest.py | 21 +++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) rename test-data-generation/src/test_data_generation/deck_configuration/strategy/{final_strategies.py => deck_configuration_strategies.py} (88%) create mode 100644 test-data-generation/tests/conftest.py diff --git a/test-data-generation/src/test_data_generation/deck_configuration/datashapes.py b/test-data-generation/src/test_data_generation/deck_configuration/datashapes.py index 94cf907e308..2bf2fbb110e 100644 --- a/test-data-generation/src/test_data_generation/deck_configuration/datashapes.py +++ b/test-data-generation/src/test_data_generation/deck_configuration/datashapes.py @@ -111,14 +111,14 @@ def __str__(self) -> str: return f"{(self.row + self.col).center(self.contents.longest_string())}{self.contents}" @property - def __label(self) -> SlotName: + def label(self) -> SlotName: """Return the slot label.""" return typing.cast(SlotName, f"{self.row}{self.col}") @property def slot_label_string(self) -> str: """Return the slot label.""" - return f"{self.__label.center(self.contents.longest_string())}" + return f"{self.label.center(self.contents.longest_string())}" @property def contents_string(self) -> str: diff --git a/test-data-generation/src/test_data_generation/deck_configuration/strategy/final_strategies.py b/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py similarity index 88% rename from test-data-generation/src/test_data_generation/deck_configuration/strategy/final_strategies.py rename to test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py index 9bf70180f96..ca0ef871c69 100644 --- a/test-data-generation/src/test_data_generation/deck_configuration/strategy/final_strategies.py +++ b/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py @@ -1,4 +1,5 @@ """Test data generation for deck configuration tests.""" +from typing import Callable, List from hypothesis import assume, strategies as st from test_data_generation.deck_configuration.datashapes import ( Column, @@ -9,6 +10,8 @@ from test_data_generation.deck_configuration.strategy.helper_strategies import a_column +DeckConfigurationStrategy = Callable[..., st.SearchStrategy[DeckConfiguration]] + def _above_or_below_is_module_or_trash(col: Column, slot: Slot) -> bool: """Return True if the deck has a module above or below the specified slot.""" @@ -79,3 +82,9 @@ def a_deck_configuration_with_invalid_fixture_in_col_2( ) return deck + + +DECK_CONFIGURATION_STRATEGIES: List[DeckConfigurationStrategy] = [ + a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker, + a_deck_configuration_with_invalid_fixture_in_col_2, +] diff --git a/test-data-generation/tests/conftest.py b/test-data-generation/tests/conftest.py new file mode 100644 index 00000000000..c08290fd031 --- /dev/null +++ b/test-data-generation/tests/conftest.py @@ -0,0 +1,21 @@ +"""Pytest configuration file. + +Contains hypothesis settings profiles. +""" + +from hypothesis import settings, Verbosity + + +settings.register_profile( + "dev", + max_examples=10, + verbosity=Verbosity.normal, + deadline=None, +) + +settings.register_profile( + "ci", + max_examples=1000, + verbosity=Verbosity.verbose, + deadline=None, +) From ca00fcead5f51d6e4266c29f89527b595450164f Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Tue, 7 May 2024 06:37:55 -0700 Subject: [PATCH 05/16] packages and Makefile changes --- test-data-generation/Makefile | 17 ++-- test-data-generation/Pipfile | 6 +- test-data-generation/Pipfile.lock | 127 +++++++++++++++++++++++++++--- 3 files changed, 131 insertions(+), 19 deletions(-) diff --git a/test-data-generation/Makefile b/test-data-generation/Makefile index a4818b00ab1..1ce4889ab91 100644 --- a/test-data-generation/Makefile +++ b/test-data-generation/Makefile @@ -27,11 +27,18 @@ wheel: $(python) setup.py $(wheel_opts) bdist_wheel rm -rf build -.PHONY: test -test: - $(pytest) tests \ +.PHONY: debug-test +debug-test: + $(pytest) ./tests \ + -vvv \ -s \ --hypothesis-show-statistics \ - --hypothesis-verbosity=normal \ --hypothesis-explain \ - -vvv \ No newline at end of file + --hypothesis-profile=dev + + +.PHONY: test +test: + $(pytest) ./tests \ + --hypothesis-explain \ + --hypothesis-profile=ci \ No newline at end of file diff --git a/test-data-generation/Pipfile b/test-data-generation/Pipfile index 758bcddacb7..70da23f28f5 100644 --- a/test-data-generation/Pipfile +++ b/test-data-generation/Pipfile @@ -4,7 +4,8 @@ url = "https://pypi.org/simple" verify_ssl = true [packages] -pytest = "==7.4.3" +pytest = "==7.4.4" +pytest-asyncio = "~=0.23.0" black = "==23.11.0" mypy = "==1.7.1" flake8 = "==7.0.0" @@ -13,8 +14,9 @@ flake8-docstrings = "~=1.7.0" flake8-noqa = "~=1.4.0" hypothesis = "==6.96.1" opentrons-shared-data = {file = "../shared-data/python", editable = true} +opentrons = { editable = true, path = "../api"} test-data-generation = {file = ".", editable = true} - +astor = "0.8.1" [requires] python_version = "3.10" diff --git a/test-data-generation/Pipfile.lock b/test-data-generation/Pipfile.lock index 1b223033d61..f43daa84809 100644 --- a/test-data-generation/Pipfile.lock +++ b/test-data-generation/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1df89f797a19f2c0febc582e7452a52858511cece041f9f612a59d35628226c2" + "sha256": "149f388d38898e580ae235ebf800a3959e1018e27ceef1d12612efc5f6bad328" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,30 @@ ] }, "default": { + "aionotify": { + "hashes": [ + "sha256:385e1becfaac2d9f4326673033d53912ef9565b6febdedbec593ee966df392c6", + "sha256:64b702ad0eb115034533f9f62730a9253b79f5ff0fbf3d100c392124cdf12507" + ], + "version": "==0.2.0" + }, + "anyio": { + "hashes": [ + "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780", + "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5" + ], + "markers": "python_version >= '3.7'", + "version": "==3.7.1" + }, + "astor": { + "hashes": [ + "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", + "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.8.1" + }, "attrs": { "hashes": [ "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", @@ -110,6 +134,14 @@ "markers": "python_version >= '3.8'", "version": "==6.96.1" }, + "idna": { + "hashes": [ + "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", + "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" + ], + "markers": "python_version >= '3.5'", + "version": "==3.7" + }, "iniconfig": { "hashes": [ "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", @@ -176,10 +208,57 @@ "markers": "python_version >= '3.5'", "version": "==1.0.0" }, + "numpy": { + "hashes": [ + "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", + "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", + "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", + "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", + "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", + "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a", + "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea", + "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c", + "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", + "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", + "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be", + "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", + "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a", + "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", + "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", + "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd", + "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c", + "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", + "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0", + "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c", + "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", + "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", + "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0", + "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6", + "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2", + "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", + "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30", + "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", + "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5", + "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07", + "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", + "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4", + "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764", + "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", + "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3", + "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f" + ], + "markers": "python_version >= '3.9'", + "version": "==1.26.4" + }, + "opentrons": { + "editable": true, + "markers": "python_version >= '3.10'", + "path": "../api" + }, "opentrons-shared-data": { "editable": true, "file": "../shared-data/python", - "markers": "python_version >= '3.8'" + "markers": "python_version >= '3.10'" }, "packaging": { "hashes": [ @@ -199,19 +278,19 @@ }, "platformdirs": { "hashes": [ - "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068", - "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768" + "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf", + "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1" ], "markers": "python_version >= '3.8'", - "version": "==4.2.0" + "version": "==4.2.1" }, "pluggy": { "hashes": [ - "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981", - "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be" + "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", + "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669" ], "markers": "python_version >= '3.8'", - "version": "==1.4.0" + "version": "==1.5.0" }, "pycodestyle": { "hashes": [ @@ -317,14 +396,38 @@ "markers": "python_version >= '3.8'", "version": "==0.20.0" }, + "pyserial": { + "hashes": [ + "sha256:3c77e014170dfffbd816e6ffc205e9842efb10be9f58ec16d3e8675b4925cddb", + "sha256:c4451db6ba391ca6ca299fb3ec7bae67a5c55dde170964c7a14ceefec02f2cf0" + ], + "version": "==3.5" + }, "pytest": { "hashes": [ - "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac", - "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5" + "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280", + "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==7.4.3" + "version": "==7.4.4" + }, + "pytest-asyncio": { + "hashes": [ + "sha256:68516fdd1018ac57b846c9846b954f0393b26f094764a28c955eabb0536a4e8a", + "sha256:ffe523a89c1c222598c76856e76852b787504ddb72dd5d9b6617ffa8aa2cde5f" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==0.23.6" + }, + "sniffio": { + "hashes": [ + "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", + "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc" + ], + "markers": "python_version >= '3.7'", + "version": "==1.3.1" }, "snowballstemmer": { "hashes": [ @@ -357,7 +460,7 @@ "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0", "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a" ], - "markers": "python_version < '3.11'", + "markers": "python_version >= '3.8'", "version": "==4.11.0" } }, From 524a57e6c35cd4bd66f9de35fe1ed8282bc7c68c Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Tue, 7 May 2024 13:58:39 -0700 Subject: [PATCH 06/16] add pipettes and calls to loaded entities --- .../python_protocol_generation/ast_helpers.py | 52 +++++++- .../generation_phases/call_phase.py | 38 ++++++ .../generation_phases/load_phase.py | 121 ++++++++++-------- .../generation_phases/setup_phase.py | 5 +- .../python_protocol_generator.py | 48 ++++++- .../python_protocol_generation/util.py | 42 ++++++ 6 files changed, 243 insertions(+), 63 deletions(-) create mode 100644 test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/call_phase.py create mode 100644 test-data-generation/src/test_data_generation/python_protocol_generation/util.py diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py b/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py index bba67650563..ad3d8c3eceb 100644 --- a/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py @@ -6,6 +6,7 @@ import typing import ast from dataclasses import dataclass +from test_data_generation.python_protocol_generation.util import ProtocolContextMethods class CanGenerateAST(typing.Protocol): @@ -33,11 +34,41 @@ def generate_ast(self) -> ast.ImportFrom: @dataclass -class CallFunction: +class BaseCall: """Class to represent a method or function call.""" call_on: str - method_name: str + what_to_call: ProtocolContextMethods | str + + def _evaluate_what_to_call(self) -> str: + """Evaluate the value of what_to_call.""" + if isinstance(self.what_to_call, ProtocolContextMethods): + return self.what_to_call.value + else: + return self.what_to_call + + def generate_ast(self) -> ast.Call: + """Generate an AST node for the call.""" + what_to_call = ( + self.what_to_call.value + if isinstance(self.what_to_call, ProtocolContextMethods) + else self.what_to_call + ) + return ast.Call( + func=ast.Attribute( + value=ast.Name(id=self.call_on, ctx=ast.Load()), + attr=what_to_call, + ctx=ast.Load(), + ), + args=[ast.Constant(str_arg) for str_arg in self.args], + keywords=[], + ) + + +@dataclass +class CallFunction(BaseCall): + """Class to represent a method or function call.""" + args: typing.List[str] def generate_ast(self) -> ast.Call: @@ -45,7 +76,7 @@ def generate_ast(self) -> ast.Call: return ast.Call( func=ast.Attribute( value=ast.Name(id=self.call_on, ctx=ast.Load()), - attr=self.method_name, + attr=self._evaluate_what_to_call(), ctx=ast.Load(), ), args=[ast.Constant(str_arg) for str_arg in self.args], @@ -53,6 +84,21 @@ def generate_ast(self) -> ast.Call: ) +@dataclass +class CallAttribute(BaseCall): + """Class to represent a method or function call.""" + + def generate_ast(self) -> ast.Call: + """Generate an AST node for the call.""" + return ast.Expr( + value=ast.Attribute( + value=ast.Name(id=self.call_on, ctx=ast.Load()), + attr=self._evaluate_what_to_call(), + ctx=ast.Load(), + ) + ) + + @dataclass class AssignStatement: """Class to represent an assignment statement.""" diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/call_phase.py b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/call_phase.py new file mode 100644 index 00000000000..d5ea3fab861 --- /dev/null +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/call_phase.py @@ -0,0 +1,38 @@ +"""This module contains functions that make calls against the various load statements in a protocol. + +Example load statements: load_module, load_labware, load_waste_chute, load_pipette, etc. +Example calls: module.labware, waste_chute.top, etc. +This is required to ensure that the loaded entities are recognized by the analysis engine. +""" +import typing +from test_data_generation.python_protocol_generation import ast_helpers as ast_h +from test_data_generation.python_protocol_generation.util import ProtocolContextMethods + + +def create_call_to_attribute_on_loaded_entity( + load_statement: ast_h.AssignStatement, +) -> ast_h.CallFunction: + """Create a call statement from a load statement.""" + assert isinstance(load_statement.value, ast_h.CallFunction) + + if load_statement.value.what_to_call in [ + ProtocolContextMethods.LOAD_WASTE_CHUTE, + ProtocolContextMethods.LOAD_TRASH_BIN, + ]: + what_to_call = "location" + else: + what_to_call = "api_version" + + return ast_h.CallAttribute( + call_on=load_statement.var_name, + what_to_call=what_to_call, + ) + + +def create_calls_to_loaded_entities( + load_statements: typing.List[ast_h.AssignStatement], +) -> typing.List[ast_h.CallFunction]: + """Create calls to loaded entity from .""" + return [ + create_call_to_attribute_on_loaded_entity(entity) for entity in load_statements + ] diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py index b78495658eb..53cbfd6ed6a 100644 --- a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py @@ -11,6 +11,12 @@ RowName, ) from test_data_generation.python_protocol_generation import ast_helpers as ast_h +from test_data_generation.python_protocol_generation.util import PipetteConfiguration +from test_data_generation.python_protocol_generation.util import ( + ModuleNames, + ProtocolContextMethods, + PROTOCOL_CONTEXT_VAR_NAME, +) def _staging_area(row: RowName) -> ast_h.AssignStatement: @@ -24,16 +30,14 @@ def _staging_area(row: RowName) -> ast_h.AssignStatement: return ast_h.AssignStatement( var_name=f"well_plate_{row}4", value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_labware", + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_LABWARE, args=[labware_name, labware_location], ), ) -def _waste_chute( - has_staging_area: bool, has_cover: bool -) -> typing.List[ast_h.AssignStatement]: +def _waste_chute(has_staging_area: bool) -> typing.List[ast_h.AssignStatement]: """Create a waste chute. If has_staging_area is True, a staging area is created in row D. @@ -42,15 +46,13 @@ def _waste_chute( ast_h.AssignStatement( var_name="waste_chute", value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_waste_chute", + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_WASTE_CHUTE, args=[], ), ) ] - # TODO: If has_cover, set USE_96_CHANNEL_PIPETTE to False - if has_staging_area: entries.append(_staging_area("d")) @@ -59,18 +61,9 @@ def _waste_chute( def _magnetic_block_on_staging_area(row: RowName) -> typing.List[ast_h.AssignStatement]: """Create a magnetic block on a staging area in a specified row.""" - module_name = "magneticBlockV1" - module_location = f"{row.upper()}3" - + slot = typing.cast(SlotName, f"{row}3") entries = [ - ast_h.AssignStatement( - var_name=f"mag_block_{row}3", - value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_module", - args=[module_name, module_location], - ), - ), + _magnetic_block(slot), _staging_area(row), ] return entries @@ -85,8 +78,8 @@ def _trash_bin(slot: SlotName) -> ast_h.AssignStatement: return ast_h.AssignStatement( var_name=f"trash_bin_{slot}", value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_trash_bin", + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_TRASH_BIN, args=[location], ), ) @@ -96,14 +89,12 @@ def _trash_bin(slot: SlotName) -> ast_h.AssignStatement: def _thermocycler_module() -> ast_h.AssignStatement: """Create a thermocycler module.""" - module_name = "thermocyclerModuleV2" - return ast_h.AssignStatement( var_name="thermocycler_module", value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_module", - args=[module_name], + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_MODULE, + args=[ModuleNames.THERMOCYCLER_MODULE.value], ), ) @@ -112,14 +103,13 @@ def _thermocycler_module() -> ast_h.AssignStatement: def _temperature_module(slot: SlotName) -> ast_h.AssignStatement: """Create a temperature module in a specified slot.""" - module_name = "temperatureModuleV2" module_location = slot.upper() return ast_h.AssignStatement( var_name=f"temperature_module_{slot}", value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_module", - args=[module_name, module_location], + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_MODULE, + args=[ModuleNames.TEMPERATURE_MODULE.value, module_location], ), ) @@ -128,14 +118,13 @@ def _temperature_module(slot: SlotName) -> ast_h.AssignStatement: def _magnetic_block(slot_name: SlotName) -> ast_h.AssignStatement: """Create a magnetic block in a specified slot.""" - module_name = "magneticBlockV1" module_location = slot_name.upper() return ast_h.AssignStatement( var_name=f"mag_block_{slot_name}", value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_module", - args=[module_name, module_location], + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_MODULE, + args=[ModuleNames.MAGNETIC_BLOCK_MODULE.value, module_location], ), ) # Call module.labware to make sure it is included as part of the analysis @@ -143,15 +132,14 @@ def _magnetic_block(slot_name: SlotName) -> ast_h.AssignStatement: def _heater_shaker_module(slot_name: SlotName) -> ast_h.AssignStatement: """Create a heater shaker module in a specified slot.""" - module_name = "heaterShakerModuleV1" module_location = slot_name.upper() return ast_h.AssignStatement( var_name=f"heater_shaker_{slot_name}", value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_module", - args=[module_name, module_location], + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_MODULE, + args=[ModuleNames.HEATER_SHAKER_MODULE.value, module_location], ), ) # Call module.labware to make sure it is included as part of the analysis @@ -165,30 +153,24 @@ def _labware_slot(slot_name: SlotName) -> ast_h.AssignStatement: return ast_h.AssignStatement( var_name=f"well_plate_{slot_name}", value=ast_h.CallFunction( - call_on="protocol_context", - method_name="load_labware", + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_LABWARE, args=[labware_name, labware_location], ), ) # well_plate_{slot}.is_tiprack -def create_load_statement( +def create_deck_slot_load_statement( slot: Slot, ) -> ast_h.AssignStatement | typing.List[ast_h.AssignStatement]: """Maps the contents of a slot to the correct assign statement.""" match slot.contents: - case PSC.WASTE_CHUTE: - return _waste_chute(False, True) - - case PSC.WASTE_CHUTE_NO_COVER: - return _waste_chute(False, False) + case PSC.WASTE_CHUTE | PSC.WASTE_CHUTE_NO_COVER: + return _waste_chute(False) - case PSC.STAGING_AREA_WITH_WASTE_CHUTE: - return _waste_chute(True, True) - - case PSC.STAGING_AREA_WITH_WASTE_CHUTE_NO_COVER: - return _waste_chute(True, False) + case PSC.STAGING_AREA_WITH_WASTE_CHUTE | PSC.STAGING_AREA_WITH_WASTE_CHUTE_NO_COVER: + return _waste_chute(True) case PSC.STAGING_AREA_WITH_MAGNETIC_BLOCK: return _magnetic_block_on_staging_area(slot.row) @@ -218,15 +200,46 @@ def create_load_statement( raise (ValueError(f"Unknown slot contents: {slot.contents}")) -def create_load_statements( +def create_deck_slot_load_statements( slots: typing.List[Slot], ) -> typing.List[ast_h.AssignStatement]: """Iterates over a list of slots and creates the corresponding load statements.""" entries: typing.List[ast_h.AssignStatement] = [] for slot in slots: - load_statement = create_load_statement(slot) + load_statement = create_deck_slot_load_statement(slot) if isinstance(load_statement, typing.List): entries.extend(load_statement) else: entries.append(load_statement) return entries + + +def create_pipette_load_statements( + pipette_config: PipetteConfiguration, +) -> typing.List[ast_h.AssignStatement]: + """Create the load statements for a pipette configuration.""" + entries: typing.List[ast_h.AssignStatement] = [] + if pipette_config.left is not None: + entries.append( + ast_h.AssignStatement( + var_name="left_pipette", + value=ast_h.CallFunction( + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_INSTRUMENT, + args=[pipette_config.left.value], + ), + ) + ) + if pipette_config.right is not None: + entries.append( + ast_h.AssignStatement( + var_name="right_pipette", + value=ast_h.CallFunction( + call_on=PROTOCOL_CONTEXT_VAR_NAME, + what_to_call=ProtocolContextMethods.LOAD_INSTRUMENT, + args=[pipette_config.right.value], + ), + ) + ) + + return entries diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/setup_phase.py b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/setup_phase.py index f065cbe302c..f18047f975a 100644 --- a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/setup_phase.py +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/setup_phase.py @@ -2,6 +2,9 @@ import ast import typing from test_data_generation.python_protocol_generation import ast_helpers as ast_h +from test_data_generation.python_protocol_generation.util import ( + PROTOCOL_CONTEXT_VAR_NAME, +) def create_requirements_dict( @@ -30,5 +33,5 @@ def create_protocol_context_run_function() -> ast_h.FunctionDefinition: """Create a function definition for the run function of a protocol.""" return ast_h.FunctionDefinition( name="run", - args=["protocol_context"], + args=[PROTOCOL_CONTEXT_VAR_NAME], ) diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/python_protocol_generator.py b/test-data-generation/src/test_data_generation/python_protocol_generation/python_protocol_generator.py index 113f1ccb9f2..07b433a6d8d 100644 --- a/test-data-generation/src/test_data_generation/python_protocol_generation/python_protocol_generator.py +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/python_protocol_generator.py @@ -4,13 +4,22 @@ import astor # type: ignore import typing from .ast_helpers import CanGenerateAST -from test_data_generation.deck_configuration.datashapes import DeckConfiguration -from .generation_phases.load_phase import create_load_statements +from test_data_generation.deck_configuration.datashapes import ( + DeckConfiguration, + PossibleSlotContents as PSC, +) + from .generation_phases.setup_phase import ( create_protocol_context_run_function, import_protocol_context, create_requirements_dict, ) +from .generation_phases.load_phase import ( + create_deck_slot_load_statements, + create_pipette_load_statements, +) +from .generation_phases.call_phase import create_calls_to_loaded_entities +from .util import PipetteConfiguration, PipetteNames class PythonProtocolGenerator: @@ -35,15 +44,44 @@ def __init__( ] ) + self._pipettes = self._choose_pipettes() + + def _choose_pipettes(self) -> PipetteConfiguration: + """Choose the pipettes to use based on the deck configuration.""" + if self._deck_configuration.d.col3.contents.is_one_of( + [PSC.WASTE_CHUTE_NO_COVER, PSC.STAGING_AREA_WITH_WASTE_CHUTE_NO_COVER] + ): + return PipetteConfiguration( + left=PipetteNames.NINETY_SIX_CHANNEL, right=None + ) + else: + return PipetteConfiguration( + left=PipetteNames.SINGLE_CHANNEL, right=PipetteNames.MULTI_CHANNEL + ) + def generate_protocol(self) -> str: """Generate the Python protocol code.""" module = ast.Module( body=[statement.generate_ast() for statement in self._top_level_statements] ) run_function = create_protocol_context_run_function().generate_ast() - load_entries = create_load_statements(self._deck_configuration.slots) - for entry in load_entries: - run_function.body.append(entry.generate_ast()) + pipette_load_statements = create_pipette_load_statements(self._pipettes) + deck_slot_load_statements = create_deck_slot_load_statements( + self._deck_configuration.slots + ) + + calls_to_loaded_entities = create_calls_to_loaded_entities( + pipette_load_statements + deck_slot_load_statements + ) + + statements_to_make = ( + pipette_load_statements + + deck_slot_load_statements + + calls_to_loaded_entities + ) + + for statement in statements_to_make: + run_function.body.append(statement.generate_ast()) module.body.append(run_function) return str(astor.to_source(module)) diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/util.py b/test-data-generation/src/test_data_generation/python_protocol_generation/util.py new file mode 100644 index 00000000000..e0516d0f1dc --- /dev/null +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/util.py @@ -0,0 +1,42 @@ +"""Constants and datashapes used in the protocol generation.""" + +import dataclasses +import typing +import enum + +PROTOCOL_CONTEXT_VAR_NAME: typing.Final[str] = "protocol_context" + + +class PipetteNames(str, enum.Enum): + """Names of the pipettes used in the protocol.""" + + SINGLE_CHANNEL = "flex_1channel_1000" + MULTI_CHANNEL = "flex_8channel_1000" + NINETY_SIX_CHANNEL = "flex_96channel_1000" + + +@dataclasses.dataclass +class PipetteConfiguration: + """Configuration for a pipette.""" + + left: PipetteNames | None + right: PipetteNames | None + + +class ModuleNames(str, enum.Enum): + """Names of the modules used in the protocol.""" + + MAGNETIC_BLOCK_MODULE = "magneticBlockV1" + THERMOCYCLER_MODULE = "thermocyclerModuleV2" + TEMPERATURE_MODULE = "temperatureModuleV2" + HEATER_SHAKER_MODULE = "heaterShakerModuleV1" + + +class ProtocolContextMethods(str, enum.Enum): + """Methods available on the protocol context.""" + + LOAD_MODULE = "load_module" + LOAD_LABWARE = "load_labware" + LOAD_INSTRUMENT = "load_instrument" + LOAD_WASTE_CHUTE = "load_waste_chute" + LOAD_TRASH_BIN = "load_trash_bin" From ad5059c7eacb3dfebab7dda6de1faed6544abcf4 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Tue, 7 May 2024 13:59:07 -0700 Subject: [PATCH 07/16] fix bad logic that was causing configurations to be skipped --- .../strategy/helper_strategies.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py b/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py index 17950f63a39..9796d01b319 100644 --- a/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py +++ b/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py @@ -6,14 +6,16 @@ Row, Slot, PossibleSlotContents as PSC, + RowName, + ColumnName, ) @st.composite def a_slot( draw: st.DrawFn, - row: str, - col: str, + row: RowName, + col: ColumnName, content_options: List[PSC] = PSC.all(), ) -> Slot: """Generate a slot with a random content. @@ -39,7 +41,7 @@ def a_slot( if not content.is_a_staging_area() ] - if col == "1" and (row == "A" or row == "B"): + if col == "1" and (row == "a" or row == "b"): return draw( st.builds( Slot, @@ -49,7 +51,7 @@ def a_slot( ) ) elif col == "3": - if row == "D": + if row == "d": return draw( st.builds( Slot, @@ -83,7 +85,7 @@ def a_slot( @st.composite def a_row( draw: st.DrawFn, - row: str, + row: RowName, content_options: List[PSC] = PSC.all(), ) -> Row: """Generate a row with random slots.""" @@ -101,7 +103,7 @@ def a_row( @st.composite def a_column( draw: st.DrawFn, - col: str, + col: ColumnName, content_options: List[PSC] = PSC.all(), ) -> Column: """Generate a column with random slots.""" From 189495bd1334923bdf0ef7be4f540f954c694887 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Wed, 8 May 2024 06:27:30 -0700 Subject: [PATCH 08/16] misc --- .../strategy/deck_configuration_strategies.py | 13 ++++++++----- test-data-generation/tests/conftest.py | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py b/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py index ca0ef871c69..d5f2386b685 100644 --- a/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py +++ b/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py @@ -1,5 +1,5 @@ """Test data generation for deck configuration tests.""" -from typing import Callable, List +from typing import Callable, Dict, List from hypothesis import assume, strategies as st from test_data_generation.deck_configuration.datashapes import ( Column, @@ -84,7 +84,10 @@ def a_deck_configuration_with_invalid_fixture_in_col_2( return deck -DECK_CONFIGURATION_STRATEGIES: List[DeckConfigurationStrategy] = [ - a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker, - a_deck_configuration_with_invalid_fixture_in_col_2, -] +DECK_CONFIGURATION_STRATEGIES: Dict[str, DeckConfigurationStrategy] = { + f.__name__: f + for f in [ + a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker, + a_deck_configuration_with_invalid_fixture_in_col_2, + ] +} diff --git a/test-data-generation/tests/conftest.py b/test-data-generation/tests/conftest.py index c08290fd031..39e39ae66e9 100644 --- a/test-data-generation/tests/conftest.py +++ b/test-data-generation/tests/conftest.py @@ -3,7 +3,7 @@ Contains hypothesis settings profiles. """ -from hypothesis import settings, Verbosity +from hypothesis import settings, Verbosity, Phase settings.register_profile( @@ -11,6 +11,7 @@ max_examples=10, verbosity=Verbosity.normal, deadline=None, + phases=(Phase.explicit, Phase.reuse, Phase.generate, Phase.target), ) settings.register_profile( From 9d2a8a93f492dcb1b201c40ae85e8c80760280d4 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Wed, 8 May 2024 08:19:12 -0700 Subject: [PATCH 09/16] fix: handle if thermocycler setting both slots a1 and b1 --- .../strategy/deck_configuration_strategies.py | 33 +++--- .../strategy/helper_strategies.py | 106 ++++++++++++++---- 2 files changed, 100 insertions(+), 39 deletions(-) diff --git a/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py b/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py index d5f2386b685..14fd0814ff3 100644 --- a/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py +++ b/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py @@ -8,7 +8,10 @@ PossibleSlotContents as PSC, ) -from test_data_generation.deck_configuration.strategy.helper_strategies import a_column +from test_data_generation.deck_configuration.strategy.helper_strategies import ( + a_column, + a_deck_by_columns, +) DeckConfigurationStrategy = Callable[..., st.SearchStrategy[DeckConfiguration]] @@ -29,14 +32,7 @@ def a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_sha ) -> DeckConfiguration: """Generate a deck with a module or trash bin fixture above or below a heater shaker.""" deck = draw( - st.builds( - DeckConfiguration.from_cols, - col1=a_column("1"), - col2=a_column( - "2", content_options=[PSC.LABWARE_SLOT, PSC.MAGNETIC_BLOCK_MODULE] - ), - col3=a_column("3"), - ) + a_deck_by_columns(col_2_contents=[PSC.LABWARE_SLOT, PSC.MAGNETIC_BLOCK_MODULE]) ) column = deck.column_by_number(draw(st.sampled_from(["1", "3"]))) @@ -66,21 +62,18 @@ def a_deck_configuration_with_invalid_fixture_in_col_2( PSC.TRASH_BIN, PSC.TEMPERATURE_MODULE, ] - column2 = draw(a_column("2", content_options=POSSIBLE_FIXTURES)) + + deck = draw(a_deck_by_columns(col_2_contents=POSSIBLE_FIXTURES)) + num_invalid_fixtures = len( - [True for slot in column2.slots if slot.contents.is_one_of(INVALID_FIXTURES)] + [ + True + for slot in deck.column_by_number("2").slots + if slot.contents.is_one_of(INVALID_FIXTURES) + ] ) assume(num_invalid_fixtures > 0) - deck = draw( - st.builds( - DeckConfiguration.from_cols, - col1=a_column("1"), - col2=st.just(column2), - col3=a_column("3"), - ) - ) - return deck diff --git a/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py b/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py index 9796d01b319..d9a5b0375a9 100644 --- a/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py +++ b/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py @@ -1,11 +1,12 @@ """Test data generation for deck configuration tests.""" -from typing import List +import typing from hypothesis import strategies as st from test_data_generation.deck_configuration.datashapes import ( Column, Row, Slot, PossibleSlotContents as PSC, + DeckConfiguration, RowName, ColumnName, ) @@ -16,7 +17,8 @@ def a_slot( draw: st.DrawFn, row: RowName, col: ColumnName, - content_options: List[PSC] = PSC.all(), + thermocycler_on_deck: bool, + content_options: typing.List[PSC] = PSC.all(), ) -> Slot: """Generate a slot with a random content. @@ -26,12 +28,6 @@ def a_slot( no_thermocycler = [ content for content in content_options if content is not PSC.THERMOCYCLER_MODULE ] - no_waste_chute_or_staging_area = [ - content - for content in content_options - if not content.is_a_waste_chute() and not content.is_a_staging_area() - ] - no_waste_chute_or_thermocycler = [ content for content in no_thermocycler if not content.is_a_waste_chute() ] @@ -41,13 +37,13 @@ def a_slot( if not content.is_a_staging_area() ] - if col == "1" and (row == "a" or row == "b"): + if thermocycler_on_deck and col == "1" and (row == "a" or row == "b"): return draw( st.builds( Slot, row=st.just(row), col=st.just(col), - contents=st.sampled_from(no_waste_chute_or_staging_area), + contents=st.just(PSC.THERMOCYCLER_MODULE), ) ) elif col == "3": @@ -86,16 +82,32 @@ def a_slot( def a_row( draw: st.DrawFn, row: RowName, - content_options: List[PSC] = PSC.all(), + thermocycler_on_deck: bool, + content_options: typing.List[PSC] = PSC.all(), ) -> Row: """Generate a row with random slots.""" return draw( st.builds( Row, row=st.just(row), - col1=a_slot(row=row, col="1", content_options=content_options), - col2=a_slot(row=row, col="2", content_options=content_options), - col3=a_slot(row=row, col="3", content_options=content_options), + col1=a_slot( + row=row, + col="1", + thermocycler_on_deck=thermocycler_on_deck, + content_options=content_options, + ), + col2=a_slot( + row=row, + col="2", + thermocycler_on_deck=thermocycler_on_deck, + content_options=content_options, + ), + col3=a_slot( + row=row, + col="3", + thermocycler_on_deck=thermocycler_on_deck, + content_options=content_options, + ), ) ) @@ -104,16 +116,72 @@ def a_row( def a_column( draw: st.DrawFn, col: ColumnName, - content_options: List[PSC] = PSC.all(), + thermocycler_on_deck: bool, + content_options: typing.List[PSC] = PSC.all(), ) -> Column: """Generate a column with random slots.""" return draw( st.builds( Column, col=st.just(col), - a=a_slot(row="a", col=col, content_options=content_options), - b=a_slot(row="b", col=col, content_options=content_options), - c=a_slot(row="c", col=col, content_options=content_options), - d=a_slot(row="d", col=col, content_options=content_options), + a=a_slot( + row="a", + col=col, + thermocycler_on_deck=thermocycler_on_deck, + content_options=content_options, + ), + b=a_slot( + row="b", + col=col, + thermocycler_on_deck=thermocycler_on_deck, + content_options=content_options, + ), + c=a_slot( + row="c", + col=col, + thermocycler_on_deck=thermocycler_on_deck, + content_options=content_options, + ), + d=a_slot( + row="d", + col=col, + thermocycler_on_deck=thermocycler_on_deck, + content_options=content_options, + ), + ) + ) + + +@st.composite +def a_deck_by_columns( + draw: st.DrawFn, + thermocycler_on_deck: bool | None = None, + col_1_contents=PSC.all(), + col_2_contents=PSC.all(), + col_3_contents=PSC.all(), +) -> DeckConfiguration: + """Generate a deck by columns.""" + + if thermocycler_on_deck is None: + thermocycler_on_deck = draw(st.booleans()) + + return draw( + st.builds( + DeckConfiguration.from_cols, + a_column( + "1", + thermocycler_on_deck=thermocycler_on_deck, + content_options=col_1_contents, + ), + a_column( + "2", + thermocycler_on_deck=thermocycler_on_deck, + content_options=col_2_contents, + ), + a_column( + "3", + thermocycler_on_deck=thermocycler_on_deck, + content_options=col_3_contents, + ), ) ) From 813122df8aae2b22b3c86afad9bf9d5b25d83309 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Wed, 8 May 2024 08:20:32 -0700 Subject: [PATCH 10/16] fix: handle thermoycler statements in python protocol --- .../python_protocol_generation/generation_phases/load_phase.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py index 53cbfd6ed6a..8f1e610ff23 100644 --- a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py @@ -206,6 +206,9 @@ def create_deck_slot_load_statements( """Iterates over a list of slots and creates the corresponding load statements.""" entries: typing.List[ast_h.AssignStatement] = [] for slot in slots: + if slot.contents == PSC.THERMOCYCLER_MODULE and slot.label == "b1": + continue + load_statement = create_deck_slot_load_statement(slot) if isinstance(load_statement, typing.List): entries.extend(load_statement) From d899c27f067f20f9315c356a7f7edecc9fd06669 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Wed, 8 May 2024 08:21:12 -0700 Subject: [PATCH 11/16] fix: label pipette mounts --- .../generation_phases/load_phase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py index 8f1e610ff23..636bbf4c2f4 100644 --- a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/load_phase.py @@ -229,7 +229,7 @@ def create_pipette_load_statements( value=ast_h.CallFunction( call_on=PROTOCOL_CONTEXT_VAR_NAME, what_to_call=ProtocolContextMethods.LOAD_INSTRUMENT, - args=[pipette_config.left.value], + args=[pipette_config.left.value, "left"], ), ) ) @@ -240,7 +240,7 @@ def create_pipette_load_statements( value=ast_h.CallFunction( call_on=PROTOCOL_CONTEXT_VAR_NAME, what_to_call=ProtocolContextMethods.LOAD_INSTRUMENT, - args=[pipette_config.right.value], + args=[pipette_config.right.value, "right"], ), ) ) From cb118e449784badc85410477e3f53b8965589a60 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Thu, 9 May 2024 05:56:46 -0700 Subject: [PATCH 12/16] formatting, typing, linting --- .../strategy/deck_configuration_strategies.py | 39 ++----------------- .../strategy/helper_strategies.py | 7 ++-- .../python_protocol_generation/ast_helpers.py | 19 ++------- .../generation_phases/call_phase.py | 4 +- .../test_deck_configuration.py | 15 ------- 5 files changed, 11 insertions(+), 73 deletions(-) diff --git a/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py b/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py index 14fd0814ff3..088676399ed 100644 --- a/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py +++ b/test-data-generation/src/test_data_generation/deck_configuration/strategy/deck_configuration_strategies.py @@ -1,48 +1,16 @@ """Test data generation for deck configuration tests.""" -from typing import Callable, Dict, List +import typing from hypothesis import assume, strategies as st from test_data_generation.deck_configuration.datashapes import ( - Column, DeckConfiguration, - Slot, PossibleSlotContents as PSC, ) from test_data_generation.deck_configuration.strategy.helper_strategies import ( - a_column, a_deck_by_columns, ) -DeckConfigurationStrategy = Callable[..., st.SearchStrategy[DeckConfiguration]] - - -def _above_or_below_is_module_or_trash(col: Column, slot: Slot) -> bool: - """Return True if the deck has a module above or below the specified slot.""" - above = col.slot_above(slot) - below = col.slot_below(slot) - - return (above is not None and above.contents.is_module_or_trash_bin()) or ( - below is not None and below.contents.is_module_or_trash_bin() - ) - - -@st.composite -def a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker( - draw: st.DrawFn, -) -> DeckConfiguration: - """Generate a deck with a module or trash bin fixture above or below a heater shaker.""" - deck = draw( - a_deck_by_columns(col_2_contents=[PSC.LABWARE_SLOT, PSC.MAGNETIC_BLOCK_MODULE]) - ) - column = deck.column_by_number(draw(st.sampled_from(["1", "3"]))) - - assume(column.number_of(PSC.HEATER_SHAKER_MODULE) in [1, 2]) - for slot in column.slots: - if slot.contents is PSC.HEATER_SHAKER_MODULE: - assume(_above_or_below_is_module_or_trash(column, slot)) - deck.override_with_column(column) - - return deck +DeckConfigurationStrategy = typing.Callable[..., st.SearchStrategy[DeckConfiguration]] @st.composite @@ -77,10 +45,9 @@ def a_deck_configuration_with_invalid_fixture_in_col_2( return deck -DECK_CONFIGURATION_STRATEGIES: Dict[str, DeckConfigurationStrategy] = { +DECK_CONFIGURATION_STRATEGIES: typing.Dict[str, DeckConfigurationStrategy] = { f.__name__: f for f in [ - a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker, a_deck_configuration_with_invalid_fixture_in_col_2, ] } diff --git a/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py b/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py index d9a5b0375a9..027ce9e7620 100644 --- a/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py +++ b/test-data-generation/src/test_data_generation/deck_configuration/strategy/helper_strategies.py @@ -156,12 +156,11 @@ def a_column( def a_deck_by_columns( draw: st.DrawFn, thermocycler_on_deck: bool | None = None, - col_1_contents=PSC.all(), - col_2_contents=PSC.all(), - col_3_contents=PSC.all(), + col_1_contents: typing.List[PSC] = PSC.all(), + col_2_contents: typing.List[PSC] = PSC.all(), + col_3_contents: typing.List[PSC] = PSC.all(), ) -> DeckConfiguration: """Generate a deck by columns.""" - if thermocycler_on_deck is None: thermocycler_on_deck = draw(st.booleans()) diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py b/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py index ad3d8c3eceb..691903e04b4 100644 --- a/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/ast_helpers.py @@ -47,22 +47,9 @@ def _evaluate_what_to_call(self) -> str: else: return self.what_to_call - def generate_ast(self) -> ast.Call: + def generate_ast(self) -> ast.AST: """Generate an AST node for the call.""" - what_to_call = ( - self.what_to_call.value - if isinstance(self.what_to_call, ProtocolContextMethods) - else self.what_to_call - ) - return ast.Call( - func=ast.Attribute( - value=ast.Name(id=self.call_on, ctx=ast.Load()), - attr=what_to_call, - ctx=ast.Load(), - ), - args=[ast.Constant(str_arg) for str_arg in self.args], - keywords=[], - ) + raise NotImplementedError @dataclass @@ -88,7 +75,7 @@ def generate_ast(self) -> ast.Call: class CallAttribute(BaseCall): """Class to represent a method or function call.""" - def generate_ast(self) -> ast.Call: + def generate_ast(self) -> ast.Expr: """Generate an AST node for the call.""" return ast.Expr( value=ast.Attribute( diff --git a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/call_phase.py b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/call_phase.py index d5ea3fab861..24a35efc099 100644 --- a/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/call_phase.py +++ b/test-data-generation/src/test_data_generation/python_protocol_generation/generation_phases/call_phase.py @@ -11,7 +11,7 @@ def create_call_to_attribute_on_loaded_entity( load_statement: ast_h.AssignStatement, -) -> ast_h.CallFunction: +) -> ast_h.CallAttribute: """Create a call statement from a load statement.""" assert isinstance(load_statement.value, ast_h.CallFunction) @@ -31,7 +31,7 @@ def create_call_to_attribute_on_loaded_entity( def create_calls_to_loaded_entities( load_statements: typing.List[ast_h.AssignStatement], -) -> typing.List[ast_h.CallFunction]: +) -> typing.List[ast_h.CallAttribute]: """Create calls to loaded entity from .""" return [ create_call_to_attribute_on_loaded_entity(entity) for entity in load_statements diff --git a/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py b/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py index 07e189f0925..5d68e51015c 100644 --- a/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py +++ b/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py @@ -1,11 +1,9 @@ """Tests to ensure that the deck configuration is generated correctly.""" -import pytest from pathlib import Path from hypothesis import given, settings, HealthCheck from test_data_generation.deck_configuration.datashapes import DeckConfiguration from test_data_generation.deck_configuration.strategy.deck_configuration_strategies import ( - a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker, a_deck_configuration_with_invalid_fixture_in_col_2, ) from test_data_generation.python_protocol_generation.python_protocol_generator import ( @@ -13,19 +11,6 @@ ) -@given( - deck_config=a_deck_configuration_with_a_module_or_trash_slot_above_or_below_a_heater_shaker() -) -@settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) -@pytest.mark.asyncio -async def test_above_below_heater_shaker( - deck_config: DeckConfiguration, tmp_path: Path -) -> None: - """I hypothesize, that any deck configuration with a non-labware slot fixture above or below a heater-shaker is invalid.""" - protocol_content = PythonProtocolGenerator(deck_config, "2.18").generate_protocol() - print(protocol_content) - - @given(deck_config=a_deck_configuration_with_invalid_fixture_in_col_2()) @settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) def test_invalid_fixture_in_col_2( From 4481dc18425bd0e566713d34e01307c7d45475b1 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Thu, 9 May 2024 09:51:55 -0700 Subject: [PATCH 13/16] chore: add running and evaluating analysis --- test-data-generation/tests/__init__.py | 0 .../test_deck_configuration.py | 21 ++++++-- test-data-generation/tests/utils.py | 52 +++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 test-data-generation/tests/__init__.py create mode 100644 test-data-generation/tests/utils.py diff --git a/test-data-generation/tests/__init__.py b/test-data-generation/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py b/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py index 5d68e51015c..eca3aecfb25 100644 --- a/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py +++ b/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py @@ -1,21 +1,34 @@ """Tests to ensure that the deck configuration is generated correctly.""" from pathlib import Path -from hypothesis import given, settings, HealthCheck -from test_data_generation.deck_configuration.datashapes import DeckConfiguration + +import pytest +from hypothesis import HealthCheck, given, note, settings +from tests.utils import has_errors, make_the_failed_protocol_pretty, run_analysis + +from test_data_generation.datashapes import DeckConfiguration from test_data_generation.deck_configuration.strategy.deck_configuration_strategies import ( a_deck_configuration_with_invalid_fixture_in_col_2, ) from test_data_generation.python_protocol_generation.python_protocol_generator import ( PythonProtocolGenerator, ) +from test_data_generation.python_protocol_generation.strategy.python_protocol_generation_strategies import ( + a_protocol_that_loads_invalid_stuff_into_a_staging_area_col_3, + a_protocol_that_loads_invalid_stuff_into_a_staging_area_col_4, +) @given(deck_config=a_deck_configuration_with_invalid_fixture_in_col_2()) @settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) -def test_invalid_fixture_in_col_2( +@pytest.mark.asyncio +async def test_invalid_fixture_in_col_2( deck_config: DeckConfiguration, tmp_path: Path ) -> None: """I hypothesize, that any deck configuration that contains at least one, Heater-Shaker, Trash Bin, or Temperature module, in column 2 is invalid.""" protocol_content = PythonProtocolGenerator(deck_config, "2.18").generate_protocol() - print(protocol_content) + analysis_response = await run_analysis(protocol_content, tmp_path) + note(str(deck_config)) + assert has_errors( + analysis_response + ), "This deck configuration should cause analysis to fail." diff --git a/test-data-generation/tests/utils.py b/test-data-generation/tests/utils.py new file mode 100644 index 00000000000..9093d8d596f --- /dev/null +++ b/test-data-generation/tests/utils.py @@ -0,0 +1,52 @@ +import subprocess +import pathlib +from opentrons.cli.analyze import AnalyzeResults + + + +async def run_analysis(protocol_content: str, tmp_path: pathlib.Path) -> AnalyzeResults: + """Run the analysis on the generated protocol.""" + protocol_path = tmp_path / "protocol.py" + analysis_json_path = tmp_path / "analysis_result.json" + + protocol_path.write_text(protocol_content) + + command = [ + "python", + "-m", + "opentrons.cli", + "analyze", + str(protocol_path), + "--human-json-output", + str(analysis_json_path), + ] + + # Run the command + result = subprocess.run(command, capture_output=True, text=True) + + if result.returncode != 0: + raise Exception(f"Analysis failed: {result.stderr}\n{protocol_content}\n") + + return AnalyzeResults.parse_file(analysis_json_path) + +def make_the_failed_protocol_pretty(protocol_content: str) -> str: + """Pretty print the protocol.""" + command = [ + "black", + "-", + ] + + # Run the command + result = subprocess.run( + command, input=protocol_content, capture_output=True, text=True + ) + + if result.returncode != 0: + raise Exception(f"Black failed: {result.stderr}\n{protocol_content}\n") + + return result.stdout + + +def has_errors(analysis_response: AnalyzeResults) -> bool: + """Check if the analysis response has errors.""" + return len(analysis_response.errors) > 0 \ No newline at end of file From 0a3e0efabb2583608cf843e79cbc96d776b61f74 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Thu, 9 May 2024 09:53:13 -0700 Subject: [PATCH 14/16] some settings and package changes --- test-data-generation/Makefile | 14 ++++++++- test-data-generation/Pipfile | 1 + test-data-generation/Pipfile.lock | 19 +++++++++++- test-data-generation/tests/conftest.py | 40 ++++++++++++++++++++++---- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/test-data-generation/Makefile b/test-data-generation/Makefile index 1ce4889ab91..4b0d6138e40 100644 --- a/test-data-generation/Makefile +++ b/test-data-generation/Makefile @@ -31,14 +31,26 @@ wheel: debug-test: $(pytest) ./tests \ -vvv \ + --numprocesses=auto \ -s \ --hypothesis-show-statistics \ --hypothesis-explain \ --hypothesis-profile=dev +.PHONY: exploratory-test +exploratory-test: + $(pytest) ./tests \ + --numprocesses=auto \ + -s \ + --hypothesis-show-statistics \ + --hypothesis-explain \ + --hypothesis-profile=exploratory .PHONY: test test: $(pytest) ./tests \ - --hypothesis-explain \ + --hypothesis-show-statistics \ + --numprocesses=auto \ + --tb=short \ + --show-capture=stderr \ --hypothesis-profile=ci \ No newline at end of file diff --git a/test-data-generation/Pipfile b/test-data-generation/Pipfile index 70da23f28f5..4efa098c516 100644 --- a/test-data-generation/Pipfile +++ b/test-data-generation/Pipfile @@ -17,6 +17,7 @@ opentrons-shared-data = {file = "../shared-data/python", editable = true} opentrons = { editable = true, path = "../api"} test-data-generation = {file = ".", editable = true} astor = "0.8.1" +pytest-xdist = "3.6.1" [requires] python_version = "3.10" diff --git a/test-data-generation/Pipfile.lock b/test-data-generation/Pipfile.lock index f43daa84809..15fda691846 100644 --- a/test-data-generation/Pipfile.lock +++ b/test-data-generation/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "149f388d38898e580ae235ebf800a3959e1018e27ceef1d12612efc5f6bad328" + "sha256": "4874d951f3ae3a91e542cd14c9dca14eeb31d1caa77e39de028d17d6a21840bd" }, "pipfile-spec": 6, "requires": { @@ -89,6 +89,14 @@ "markers": "python_version < '3.11'", "version": "==1.2.1" }, + "execnet": { + "hashes": [ + "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc", + "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3" + ], + "markers": "python_version >= '3.8'", + "version": "==2.1.1" + }, "flake8": { "hashes": [ "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132", @@ -421,6 +429,15 @@ "markers": "python_version >= '3.8'", "version": "==0.23.6" }, + "pytest-xdist": { + "hashes": [ + "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7", + "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==3.6.1" + }, "sniffio": { "hashes": [ "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", diff --git a/test-data-generation/tests/conftest.py b/test-data-generation/tests/conftest.py index 39e39ae66e9..47596fe446c 100644 --- a/test-data-generation/tests/conftest.py +++ b/test-data-generation/tests/conftest.py @@ -3,20 +3,50 @@ Contains hypothesis settings profiles. """ -from hypothesis import settings, Verbosity, Phase +from datetime import timedelta + +from hypothesis import HealthCheck, Phase, Verbosity, settings + +# Hypothesis cannot shrink the DeckConfiguration object, but attempts to do so anyways. +# This causes tests to take exponentially longer, and then fail, which is less than ideal. +# So defaulting to not shrinking + +# If there start being tests which do not use the DeckConfiguration object, then this Phase setting +# can be applied prescriptively to only tests that use the DeckConfiguration object. +DONT_SHRINK = set(settings.default.phases) - {Phase.shrink} + +# The tests are slow because they are running the analysis on generated protocols, which takes longer than the default 200ms. +# The tests are also filtering a lot of examples because they are generating a lot of invalid protocols. + +# TODO: Stop generating so many invalid protocols + +ITS_GONNA_BE_SLOW = (HealthCheck.too_slow, HealthCheck.filter_too_much) + +default = settings( + suppress_health_check=ITS_GONNA_BE_SLOW, + phases=DONT_SHRINK, + verbosity=Verbosity.normal, +) + +settings.register_profile( + "exploratory", + parent=default, + max_examples=100, + deadline=None, +) settings.register_profile( "dev", + parent=default, max_examples=10, - verbosity=Verbosity.normal, deadline=None, - phases=(Phase.explicit, Phase.reuse, Phase.generate, Phase.target), ) + settings.register_profile( "ci", + parent=default, max_examples=1000, - verbosity=Verbosity.verbose, - deadline=None, + deadline=timedelta(seconds=10), ) From f6337ed469a6b52ef6292e845c758e6ae517dc97 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Thu, 9 May 2024 09:56:36 -0700 Subject: [PATCH 15/16] linting and formatting --- test-data-generation/tests/__init__.py | 1 + .../deck_configuration/test_deck_configuration.py | 9 +++------ test-data-generation/tests/utils.py | 6 ++++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test-data-generation/tests/__init__.py b/test-data-generation/tests/__init__.py index e69de29bb2d..46816ddf5e7 100644 --- a/test-data-generation/tests/__init__.py +++ b/test-data-generation/tests/__init__.py @@ -0,0 +1 @@ +"""Tests package.""" diff --git a/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py b/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py index eca3aecfb25..f2b493c047a 100644 --- a/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py +++ b/test-data-generation/tests/test_data_generation/deck_configuration/test_deck_configuration.py @@ -4,19 +4,16 @@ import pytest from hypothesis import HealthCheck, given, note, settings -from tests.utils import has_errors, make_the_failed_protocol_pretty, run_analysis +from tests.utils import has_errors, run_analysis + +from test_data_generation.deck_configuration.datashapes import DeckConfiguration -from test_data_generation.datashapes import DeckConfiguration from test_data_generation.deck_configuration.strategy.deck_configuration_strategies import ( a_deck_configuration_with_invalid_fixture_in_col_2, ) from test_data_generation.python_protocol_generation.python_protocol_generator import ( PythonProtocolGenerator, ) -from test_data_generation.python_protocol_generation.strategy.python_protocol_generation_strategies import ( - a_protocol_that_loads_invalid_stuff_into_a_staging_area_col_3, - a_protocol_that_loads_invalid_stuff_into_a_staging_area_col_4, -) @given(deck_config=a_deck_configuration_with_invalid_fixture_in_col_2()) diff --git a/test-data-generation/tests/utils.py b/test-data-generation/tests/utils.py index 9093d8d596f..604a02d518d 100644 --- a/test-data-generation/tests/utils.py +++ b/test-data-generation/tests/utils.py @@ -1,9 +1,10 @@ +"""Utility functions for the tests.""" + import subprocess import pathlib from opentrons.cli.analyze import AnalyzeResults - async def run_analysis(protocol_content: str, tmp_path: pathlib.Path) -> AnalyzeResults: """Run the analysis on the generated protocol.""" protocol_path = tmp_path / "protocol.py" @@ -29,6 +30,7 @@ async def run_analysis(protocol_content: str, tmp_path: pathlib.Path) -> Analyze return AnalyzeResults.parse_file(analysis_json_path) + def make_the_failed_protocol_pretty(protocol_content: str) -> str: """Pretty print the protocol.""" command = [ @@ -49,4 +51,4 @@ def make_the_failed_protocol_pretty(protocol_content: str) -> str: def has_errors(analysis_response: AnalyzeResults) -> bool: """Check if the analysis response has errors.""" - return len(analysis_response.errors) > 0 \ No newline at end of file + return len(analysis_response.errors) > 0 From c3c702bfed1124f434539e84ac29dfb1ba0e2d28 Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Thu, 9 May 2024 10:04:42 -0700 Subject: [PATCH 16/16] not using pytest-xdist yet --- test-data-generation/Makefile | 14 +------------- test-data-generation/Pipfile | 1 - test-data-generation/Pipfile.lock | 19 +------------------ 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/test-data-generation/Makefile b/test-data-generation/Makefile index 4b0d6138e40..1ce4889ab91 100644 --- a/test-data-generation/Makefile +++ b/test-data-generation/Makefile @@ -31,26 +31,14 @@ wheel: debug-test: $(pytest) ./tests \ -vvv \ - --numprocesses=auto \ -s \ --hypothesis-show-statistics \ --hypothesis-explain \ --hypothesis-profile=dev -.PHONY: exploratory-test -exploratory-test: - $(pytest) ./tests \ - --numprocesses=auto \ - -s \ - --hypothesis-show-statistics \ - --hypothesis-explain \ - --hypothesis-profile=exploratory .PHONY: test test: $(pytest) ./tests \ - --hypothesis-show-statistics \ - --numprocesses=auto \ - --tb=short \ - --show-capture=stderr \ + --hypothesis-explain \ --hypothesis-profile=ci \ No newline at end of file diff --git a/test-data-generation/Pipfile b/test-data-generation/Pipfile index 4efa098c516..70da23f28f5 100644 --- a/test-data-generation/Pipfile +++ b/test-data-generation/Pipfile @@ -17,7 +17,6 @@ opentrons-shared-data = {file = "../shared-data/python", editable = true} opentrons = { editable = true, path = "../api"} test-data-generation = {file = ".", editable = true} astor = "0.8.1" -pytest-xdist = "3.6.1" [requires] python_version = "3.10" diff --git a/test-data-generation/Pipfile.lock b/test-data-generation/Pipfile.lock index 15fda691846..f43daa84809 100644 --- a/test-data-generation/Pipfile.lock +++ b/test-data-generation/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4874d951f3ae3a91e542cd14c9dca14eeb31d1caa77e39de028d17d6a21840bd" + "sha256": "149f388d38898e580ae235ebf800a3959e1018e27ceef1d12612efc5f6bad328" }, "pipfile-spec": 6, "requires": { @@ -89,14 +89,6 @@ "markers": "python_version < '3.11'", "version": "==1.2.1" }, - "execnet": { - "hashes": [ - "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc", - "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3" - ], - "markers": "python_version >= '3.8'", - "version": "==2.1.1" - }, "flake8": { "hashes": [ "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132", @@ -429,15 +421,6 @@ "markers": "python_version >= '3.8'", "version": "==0.23.6" }, - "pytest-xdist": { - "hashes": [ - "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7", - "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==3.6.1" - }, "sniffio": { "hashes": [ "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2",