feat(api, robot-server): use runtime parameter files set in protocols and runs to set in-protocol values

api/src/opentrons/cli/analyze.py

@@ -248,7 +248,7 @@ async def _do_analyze(protocol_source: ProtocolSource) -> RunResult:
             protocol_source=protocol_source,
             parse_mode=ParseMode.NORMAL,
             run_time_param_values=None,
-            run_time_param_files=None,
+            run_time_param_paths=None,
         )
     except Exception as error:
         err_id = "analysis-setup-error"

api/src/opentrons/protocol_api/__init__.py

@@ -12,6 +12,7 @@
 from opentrons.protocols.parameters.exceptions import (
     RuntimeParameterRequired as RuntimeParameterRequiredError,
 )
+from opentrons.protocols.parameters.types import CSVParameter
 
 from .protocol_context import ProtocolContext
 from .deck import Deck
@@ -72,6 +73,7 @@
     "ALL",
     "OFF_DECK",
     "RuntimeParameterRequiredError",
+    "CSVParameter",
     # For internal Opentrons use only:
     "create_protocol_context",
     "ProtocolEngineCoreRequiredError",

api/src/opentrons/protocol_api/_parameter_context.py

@@ -1,5 +1,5 @@
 """Parameter context for python protocols."""
-
+import tempfile
 from typing import List, Optional, Union, Dict
 
 from opentrons.protocols.api_support.types import APIVersion
@@ -19,7 +19,7 @@
 from opentrons.protocol_engine.types import (
     RunTimeParameter,
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
     FileInfo,
 )
 
@@ -218,15 +218,15 @@ def set_parameters(
                 parameter.value = validated_value
 
     def initialize_csv_files(
-        self, run_time_param_file_overrides: CSVRunTimeParamFilesType
+        self, run_time_param_file_overrides: CSVRuntimeParamPaths
     ) -> None:
         """Initializes the files for CSV parameters.
 
         :meta private:
 
         This is intended for Opentrons internal use only and is not a guaranteed API.
         """
-        for variable_name, file_id in run_time_param_file_overrides.items():
+        for variable_name, file_path in run_time_param_file_overrides.items():
             try:
                 parameter = self._parameters[variable_name]
             except KeyError:
@@ -240,11 +240,34 @@ def initialize_csv_files(
                     f"File Id was provided for the parameter '{variable_name}',"
                     f" but '{variable_name}' is not a CSV parameter."
                 )
+            file_id = file_path.parent.name
+            file_name = file_path.name
+            with file_path.open() as csv_file:
+                contents = csv_file.read()
+
+            temporary_file = tempfile.NamedTemporaryFile("r+")
+            temporary_file.write(contents)
+            temporary_file.flush()
+
+            parameter_file = open(temporary_file.name, "r")
+            temporary_file.close()
 
-            parameter.file_info = FileInfo(id=file_id, name="")
-            # TODO (spp, 2024-07-16): set the file name and assign the file as parameter.value.
-            #  Most likely, we will be creating a temporary file copy of the original
-            #  to pass onto the protocol context
+            parameter.file_info = FileInfo(id=file_id, name=file_name)
+            parameter.value = parameter_file
+
+    def close_csv_files(self) -> None:
+        """Close all file handlers for CSV parameters.
+
+        :meta private:
+
+        This is intended for Opentrons internal use only and is not a guaranteed API.
+        """
+        for parameter in self._parameters.values():
+            if (
+                isinstance(parameter, csv_parameter_definition.CSVParameterDefinition)
+                and parameter.value is not None
+            ):
+                parameter.value.close()
 
     def export_parameters_for_analysis(self) -> List[RunTimeParameter]:
         """Exports all parameters into a protocol engine models for reporting in analysis.

api/src/opentrons/protocol_engine/types.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 from enum import Enum
 from dataclasses import dataclass
+from pathlib import Path
 from pydantic import (
     BaseModel,
     Field,
@@ -1069,3 +1070,4 @@ class CSVParameter(RTPBase):
 ]  # update value types as more RTP types are added
 
 CSVRunTimeParamFilesType = Mapping[StrictStr, StrictStr]
+CSVRuntimeParamPaths = Dict[str, Path]

api/src/opentrons/protocol_runner/protocol_runner.py

@@ -44,7 +44,7 @@
     DeckConfigurationType,
     RunTimeParameter,
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
 )
 from ..protocols.types import PythonProtocol
 
@@ -186,7 +186,7 @@ async def load(
         protocol_source: ProtocolSource,
         python_parse_mode: PythonParseMode,
         run_time_param_values: Optional[PrimitiveRunTimeParamValuesType],
-        run_time_param_files: Optional[CSVRunTimeParamFilesType],
+        run_time_param_paths: Optional[CSVRuntimeParamPaths],
     ) -> None:
         """Load a Python or JSONv5(& older) ProtocolSource into managed ProtocolEngine."""
         labware_definitions = await protocol_reader.extract_labware_definitions(
@@ -209,7 +209,7 @@ async def load(
                     protocol=protocol,
                     parameter_context=self._parameter_context,
                     run_time_param_overrides=run_time_param_values,
-                    run_time_param_file_overrides=run_time_param_files,
+                    run_time_param_file_overrides=run_time_param_paths,
                 )
             )
         else:
@@ -244,6 +244,7 @@ async def run_func() -> None:
             await self._protocol_executor.execute(
                 protocol=protocol,
                 context=context,
+                parameter_context=self._parameter_context,
                 run_time_parameters_with_overrides=run_time_parameters_with_overrides,
             )
 
@@ -254,7 +255,7 @@ async def run(  # noqa: D102
         deck_configuration: DeckConfigurationType,
         protocol_source: Optional[ProtocolSource] = None,
         run_time_param_values: Optional[PrimitiveRunTimeParamValuesType] = None,
-        run_time_param_files: Optional[CSVRunTimeParamFilesType] = None,
+        run_time_param_paths: Optional[CSVRuntimeParamPaths] = None,
         python_parse_mode: PythonParseMode = PythonParseMode.NORMAL,
     ) -> RunResult:
         # TODO(mc, 2022-01-11): move load to runner creation, remove from `run`
@@ -264,7 +265,7 @@ async def run(  # noqa: D102
                 protocol_source=protocol_source,
                 python_parse_mode=python_parse_mode,
                 run_time_param_values=run_time_param_values,
-                run_time_param_files=run_time_param_files,
+                run_time_param_paths=run_time_param_paths,
             )
 
         self.play(deck_configuration=deck_configuration)

api/src/opentrons/protocol_runner/python_protocol_wrappers.py

@@ -15,7 +15,7 @@
 from opentrons.protocol_engine import ProtocolEngine
 from opentrons.protocol_engine.types import (
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
 )
 from opentrons.protocol_reader import ProtocolSource, ProtocolFileRole
 from opentrons.util.broker import Broker
@@ -153,19 +153,24 @@ class PythonProtocolExecutor:
     async def execute(
         protocol: Protocol,
         context: ProtocolContext,
+        parameter_context: Optional[ParameterContext],
         run_time_parameters_with_overrides: Optional[Parameters],
     ) -> None:
         """Execute a PAPIv2 protocol with a given ProtocolContext in a child thread."""
         await to_thread.run_sync(
-            run_protocol, protocol, context, run_time_parameters_with_overrides
+            run_protocol,
+            protocol,
+            context,
+            parameter_context,
+            run_time_parameters_with_overrides,
         )
 
     @staticmethod
     def extract_run_parameters(
         protocol: PythonProtocol,
         parameter_context: ParameterContext,
         run_time_param_overrides: Optional[PrimitiveRunTimeParamValuesType],
-        run_time_param_file_overrides: Optional[CSVRunTimeParamFilesType],
+        run_time_param_file_overrides: Optional[CSVRuntimeParamPaths],
     ) -> Optional[Parameters]:
         """Extract the parameters defined in the protocol, overridden with values for the run."""
         return exec_add_parameters(

api/src/opentrons/protocol_runner/run_orchestrator.py

@@ -32,7 +32,7 @@
     DeckConfigurationType,
     RunTimeParameter,
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
 )
 from ..protocol_engine.error_recovery_policy import ErrorRecoveryPolicy
 
@@ -340,7 +340,7 @@ async def load(
         self,
         protocol_source: ProtocolSource,
         run_time_param_values: Optional[PrimitiveRunTimeParamValuesType],
-        run_time_param_files: Optional[CSVRunTimeParamFilesType],
+        run_time_param_paths: Optional[CSVRuntimeParamPaths],
         parse_mode: ParseMode,
     ) -> None:
         """Load a json/python protocol."""
@@ -356,7 +356,7 @@ async def load(
                 # doesn't conform to the new rules.
                 python_parse_mode=python_parse_mode,
                 run_time_param_values=run_time_param_values,
-                run_time_param_files=run_time_param_files,
+                run_time_param_paths=run_time_param_paths,
             )
 
     def get_is_okay_to_clear(self) -> bool:

api/src/opentrons/protocols/execution/execute.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Optional
 
-from opentrons.protocol_api import ProtocolContext
+from opentrons.protocol_api import ProtocolContext, ParameterContext
 from opentrons.protocol_api._parameters import Parameters
 from opentrons.protocols.execution.execute_python import exec_run
 from opentrons.protocols.execution.json_dispatchers import (
@@ -21,25 +21,35 @@
 def run_protocol(
     protocol: Protocol,
     context: ProtocolContext,
+    parameter_context: Optional[ParameterContext] = None,
     run_time_parameters_with_overrides: Optional[Parameters] = None,
 ) -> None:
     """Run a protocol.
 
     :param protocol: The :py:class:`.protocols.types.Protocol` to execute
     :param context: The protocol context to use.
+    :param parameter_context: The parameter context to use if running with runtime parameters.
     :param run_time_parameters_with_overrides: Run time parameters defined in the protocol,
         updated with the run's RTP override values. When we are running either simulate
         or execute, this will be None (until RTP is supported in cli commands)
     """
     if isinstance(protocol, PythonProtocol):
-        if protocol.api_level >= APIVersion(2, 0):
-            exec_run(
-                proto=protocol,
-                context=context,
-                run_time_parameters_with_overrides=run_time_parameters_with_overrides,
-            )
-        else:
-            raise RuntimeError(f"Unsupported python API version: {protocol.api_level}")
+        try:
+            if protocol.api_level >= APIVersion(2, 0):
+                exec_run(
+                    proto=protocol,
+                    context=context,
+                    run_time_parameters_with_overrides=run_time_parameters_with_overrides,
+                )
+            else:
+                raise RuntimeError(
+                    f"Unsupported python API version: {protocol.api_level}"
+                )
+        except Exception:
+            raise
+        finally:
+            if parameter_context is not None:
+                parameter_context.close_csv_files()
     else:
         if protocol.contents["schemaVersion"] == 3:
             ins = execute_json_v3.load_pipettes_from_json(context, protocol.contents)

api/src/opentrons/protocols/execution/execute_python.py

@@ -12,7 +12,7 @@
 from opentrons.protocols.types import PythonProtocol, MalformedPythonProtocolError
 from opentrons.protocol_engine.types import (
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
 )
 
 
@@ -71,7 +71,7 @@ def _raise_pretty_protocol_error(exception: Exception, filename: str) -> None:
 def _parse_and_set_parameters(
     parameter_context: ParameterContext,
     run_time_param_overrides: Optional[PrimitiveRunTimeParamValuesType],
-    run_time_param_file_overrides: Optional[CSVRunTimeParamFilesType],
+    run_time_param_file_overrides: Optional[CSVRuntimeParamPaths],
     new_globs: Dict[Any, Any],
     filename: str,
 ) -> Parameters:
@@ -111,7 +111,7 @@ def exec_add_parameters(
     protocol: PythonProtocol,
     parameter_context: ParameterContext,
     run_time_param_overrides: Optional[PrimitiveRunTimeParamValuesType],
-    run_time_param_file_overrides: Optional[CSVRunTimeParamFilesType],
+    run_time_param_file_overrides: Optional[CSVRuntimeParamPaths],
 ) -> Optional[Parameters]:
     """Exec the add_parameters function and get the final run time parameters with overrides."""
     new_globs: Dict[Any, Any] = {}

api/tests/opentrons/protocol_runner/test_protocol_runner.py

@@ -638,7 +638,7 @@ async def test_load_legacy_python(
         legacy_protocol_source,
         python_parse_mode=PythonParseMode.ALLOW_LEGACY_METADATA_AND_REQUIREMENTS,
         run_time_param_values=None,
-        run_time_param_files=None,
+        run_time_param_paths=None,
     )
 
     run_func_captor = matchers.Captor()
@@ -661,6 +661,7 @@ async def test_load_legacy_python(
         await python_protocol_executor.execute(
             protocol=legacy_protocol,
             context=protocol_context,
+            parameter_context=python_runner_subject._parameter_context,
             run_time_parameters_with_overrides=None,
         ),
     )
@@ -720,7 +721,7 @@ async def test_load_python_with_pe_papi_core(
         protocol_source,
         python_parse_mode=PythonParseMode.ALLOW_LEGACY_METADATA_AND_REQUIREMENTS,
         run_time_param_values=None,
-        run_time_param_files=None,
+        run_time_param_paths=None,
     )
 
     decoy.verify(protocol_engine.add_plugin(matchers.IsA(LegacyContextPlugin)), times=0)
@@ -783,7 +784,7 @@ async def test_load_legacy_json(
         legacy_protocol_source,
         python_parse_mode=PythonParseMode.ALLOW_LEGACY_METADATA_AND_REQUIREMENTS,
         run_time_param_values=None,
-        run_time_param_files=None,
+        run_time_param_paths=None,
     )
 
     run_func_captor = matchers.Captor()
@@ -804,6 +805,7 @@ async def test_load_legacy_json(
         await python_protocol_executor.execute(
             protocol=legacy_protocol,
             context=protocol_context,
+            parameter_context=None,
             run_time_parameters_with_overrides=None,
         ),
     )

api/tests/opentrons/protocol_runner/test_run_orchestrator.py

@@ -337,7 +337,7 @@ async def test_load_json(
     await json_protocol_subject.load(
         protocol_source=protocol_source,
         run_time_param_values=None,
-        run_time_param_files=None,
+        run_time_param_paths=None,
         parse_mode=ParseMode.NORMAL,
     )
 
@@ -364,15 +364,15 @@ async def test_load_python(
         protocol_source=protocol_source,
         parse_mode=ParseMode.NORMAL,
         run_time_param_values=None,
-        run_time_param_files=None,
+        run_time_param_paths=None,
     )
 
     decoy.verify(
         await mock_protocol_python_runner.load(
             protocol_source=protocol_source,
             python_parse_mode=PythonParseMode.NORMAL,
             run_time_param_values=None,
-            run_time_param_files=None,
+            run_time_param_paths=None,
         )
     )
 
@@ -396,7 +396,7 @@ async def test_load_json_raises_no_protocol(
         await live_protocol_subject.load(
             protocol_source=protocol_source,
             run_time_param_values=None,
-            run_time_param_files=None,
+            run_time_param_paths=None,
             parse_mode=ParseMode.NORMAL,
         )