feat(api, robot-server): use runtime parameter files set in protocols…

… and runs to set in-protocol values (#15855)

# Overview

Closes AUTH-419.

This PR hooks up the setting of runtime parameter files in the POST
`/runs`, `/protocols` and `/protocols/{protocolId}/analyses` to the
actual `CSVParameter` object used in the protocol context.

This file is sent via a dictionary of parameter name and file-id, so we
resolve the file-id into a `Path` by using the `data_files_directory`
and `data_files_store`. This is then passed to the `ParameterContext`
and used to open up a temporary file that contains the contents of the
actual file stored on the robot.

## Test Plan and Hands on Testing

Tested end to end via the following protocol and two CSV files

```
metadata = {
    'protocolName': 'CSV End to End Test',
}

requirements = {
    "robotType": "Flex",
    "apiLevel": "2.18"
}


def add_parameters(parameters):
    parameters.add_str(
        display_name="Pipette Name",
        variable_name="pipette",
        choices=[
            {"display_name": "Single channel 50µL", "value": "flex_1channel_50"},
            {"display_name": "Single channel 1000µL", "value": "flex_1channel_1000"},
            {"display_name": "Eight Channel 50µL", "value": "flex_8channel_50"},
            {"display_name": "Eight Channel 1000µL", "value": "flex_8channel_1000"},
        ],
        default="flex_1channel_50",
        description="What pipette to use during the protocol.",
    )
    parameters.add_csv_file(
        display_name="CSV Data",
        variable_name="csv_data",
        description="CSV file containing labware and volume information."
    )



def run(context):
    PIPETTE_NAME = context.params.pipette

    trash_bin = context.load_trash_bin('A3')
    tip_rack = context.load_labware('opentrons_flex_96_tiprack_50ul', 'D2')

    pipette = context.load_instrument(PIPETTE_NAME, mount="left", tip_racks=[tip_rack])
    csv_file_data = context.params.csv_data.parse_as_csv()

    labware_name = csv_file_data[1][0].strip()
    location = csv_file_data[1][1].strip()
    volume = float(csv_file_data[1][2])


    labware = context.load_labware(labware_name, location)
    pipette.pick_up_tip()
    pipette.aspirate(volume, labware.wells()[0].top())
    pipette.dispense(volume, labware.wells()[1].top())
    pipette.drop_tip()
```

```
Labware Name, Location, Volume
opentrons_96_wellplate_200ul_pcr_full_skirt, C1, 20
```

```
Labware Name, Location, Volume
nest_96_wellplate_100ul_pcr_full_skirt, C2, 30
```

## Changelog

- Resolve `runTimeParameterFiles` into a dictionary of `Paths`
- pass the `run_time_param_paths` all the way to the `ParameterContext`
where they can be opened as temporary file handlers
- close the file handlers upon protocol end
- Allow importing of `CSVParameter` from the `protocol_api` namespace.

## Review requests


## Risk assessment

Medium.

---------

Co-authored-by: Sanniti <[email protected]>

api/src/opentrons/cli/analyze.py

@@ -248,7 +248,7 @@ async def _do_analyze(protocol_source: ProtocolSource) -> RunResult:
             protocol_source=protocol_source,
             parse_mode=ParseMode.NORMAL,
             run_time_param_values=None,
-            run_time_param_files=None,
+            run_time_param_paths=None,
         )
     except Exception as error:
         err_id = "analysis-setup-error"

api/src/opentrons/protocol_api/__init__.py

@@ -12,6 +12,7 @@
 from opentrons.protocols.parameters.exceptions import (
     RuntimeParameterRequired as RuntimeParameterRequiredError,
 )
+from opentrons.protocols.parameters.types import CSVParameter
 
 from .protocol_context import ProtocolContext
 from .deck import Deck
@@ -74,6 +75,7 @@
     "ALL",
     "OFF_DECK",
     "RuntimeParameterRequiredError",
+    "CSVParameter",
     # For internal Opentrons use only:
     "create_protocol_context",
     "ProtocolEngineCoreRequiredError",

api/src/opentrons/protocol_api/_parameter_context.py

@@ -1,5 +1,5 @@
 """Parameter context for python protocols."""
-
+import tempfile
 from typing import List, Optional, Union, Dict
 
 from opentrons.protocols.api_support.types import APIVersion
@@ -19,7 +19,7 @@
 from opentrons.protocol_engine.types import (
     RunTimeParameter,
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
     FileInfo,
 )
 
@@ -218,15 +218,15 @@ def set_parameters(
                 parameter.value = validated_value
 
     def initialize_csv_files(
-        self, run_time_param_file_overrides: CSVRunTimeParamFilesType
+        self, run_time_param_file_overrides: CSVRuntimeParamPaths
     ) -> None:
         """Initializes the files for CSV parameters.
 
         :meta private:
 
         This is intended for Opentrons internal use only and is not a guaranteed API.
         """
-        for variable_name, file_id in run_time_param_file_overrides.items():
+        for variable_name, file_path in run_time_param_file_overrides.items():
             try:
                 parameter = self._parameters[variable_name]
             except KeyError:
@@ -240,11 +240,41 @@ def initialize_csv_files(
                     f"File Id was provided for the parameter '{variable_name}',"
                     f" but '{variable_name}' is not a CSV parameter."
                 )
+            # TODO(jbl 2024-08-02) This file opening should be moved elsewhere to provide more flexibility with files
+            #    that may be opened as non-text or non-UTF-8
+            # The parent folder in the path will be the file ID, so we can use that to resolve that here
+            file_id = file_path.parent.name
+            file_name = file_path.name
+
+            # Read the contents of the actual file
+            with file_path.open() as csv_file:
+                contents = csv_file.read()
+
+            # Open a temporary file with write permissions and write contents to that
+            temporary_file = tempfile.NamedTemporaryFile("r+")
+            temporary_file.write(contents)
+            temporary_file.flush()
+
+            # Open a new file handler for the temporary file with read-only permissions and close the other
+            parameter_file = open(temporary_file.name, "r")
+            temporary_file.close()
+
+            parameter.file_info = FileInfo(id=file_id, name=file_name)
+            parameter.value = parameter_file
+
+    def close_csv_files(self) -> None:
+        """Close all file handlers for CSV parameters.
 
-            parameter.file_info = FileInfo(id=file_id, name="")
-            # TODO (spp, 2024-07-16): set the file name and assign the file as parameter.value.
-            #  Most likely, we will be creating a temporary file copy of the original
-            #  to pass onto the protocol context
+        :meta private:
+
+        This is intended for Opentrons internal use only and is not a guaranteed API.
+        """
+        for parameter in self._parameters.values():
+            if (
+                isinstance(parameter, csv_parameter_definition.CSVParameterDefinition)
+                and parameter.value is not None
+            ):
+                parameter.value.close()
 
     def export_parameters_for_analysis(self) -> List[RunTimeParameter]:
         """Exports all parameters into a protocol engine models for reporting in analysis.

api/src/opentrons/protocol_engine/types.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 from enum import Enum
 from dataclasses import dataclass
+from pathlib import Path
 from pydantic import (
     BaseModel,
     Field,
@@ -1069,3 +1070,4 @@ class CSVParameter(RTPBase):
 ]  # update value types as more RTP types are added
 
 CSVRunTimeParamFilesType = Mapping[StrictStr, StrictStr]
+CSVRuntimeParamPaths = Dict[str, Path]

api/src/opentrons/protocol_runner/protocol_runner.py

@@ -44,7 +44,7 @@
     DeckConfigurationType,
     RunTimeParameter,
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
 )
 from ..protocols.types import PythonProtocol
 
@@ -186,7 +186,7 @@ async def load(
         protocol_source: ProtocolSource,
         python_parse_mode: PythonParseMode,
         run_time_param_values: Optional[PrimitiveRunTimeParamValuesType],
-        run_time_param_files: Optional[CSVRunTimeParamFilesType],
+        run_time_param_paths: Optional[CSVRuntimeParamPaths],
     ) -> None:
         """Load a Python or JSONv5(& older) ProtocolSource into managed ProtocolEngine."""
         labware_definitions = await protocol_reader.extract_labware_definitions(
@@ -209,7 +209,7 @@ async def load(
                     protocol=protocol,
                     parameter_context=self._parameter_context,
                     run_time_param_overrides=run_time_param_values,
-                    run_time_param_file_overrides=run_time_param_files,
+                    run_time_param_file_overrides=run_time_param_paths,
                 )
             )
         else:
@@ -244,6 +244,7 @@ async def run_func() -> None:
             await self._protocol_executor.execute(
                 protocol=protocol,
                 context=context,
+                parameter_context=self._parameter_context,
                 run_time_parameters_with_overrides=run_time_parameters_with_overrides,
             )
 
@@ -254,7 +255,7 @@ async def run(  # noqa: D102
         deck_configuration: DeckConfigurationType,
         protocol_source: Optional[ProtocolSource] = None,
         run_time_param_values: Optional[PrimitiveRunTimeParamValuesType] = None,
-        run_time_param_files: Optional[CSVRunTimeParamFilesType] = None,
+        run_time_param_paths: Optional[CSVRuntimeParamPaths] = None,
         python_parse_mode: PythonParseMode = PythonParseMode.NORMAL,
     ) -> RunResult:
         # TODO(mc, 2022-01-11): move load to runner creation, remove from `run`
@@ -264,7 +265,7 @@ async def run(  # noqa: D102
                 protocol_source=protocol_source,
                 python_parse_mode=python_parse_mode,
                 run_time_param_values=run_time_param_values,
-                run_time_param_files=run_time_param_files,
+                run_time_param_paths=run_time_param_paths,
             )
 
         self.play(deck_configuration=deck_configuration)

api/src/opentrons/protocol_runner/python_protocol_wrappers.py

@@ -15,7 +15,7 @@
 from opentrons.protocol_engine import ProtocolEngine
 from opentrons.protocol_engine.types import (
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
 )
 from opentrons.protocol_reader import ProtocolSource, ProtocolFileRole
 from opentrons.util.broker import Broker
@@ -153,19 +153,24 @@ class PythonProtocolExecutor:
     async def execute(
         protocol: Protocol,
         context: ProtocolContext,
+        parameter_context: Optional[ParameterContext],
         run_time_parameters_with_overrides: Optional[Parameters],
     ) -> None:
         """Execute a PAPIv2 protocol with a given ProtocolContext in a child thread."""
         await to_thread.run_sync(
-            run_protocol, protocol, context, run_time_parameters_with_overrides
+            run_protocol,
+            protocol,
+            context,
+            parameter_context,
+            run_time_parameters_with_overrides,
         )
 
     @staticmethod
     def extract_run_parameters(
         protocol: PythonProtocol,
         parameter_context: ParameterContext,
         run_time_param_overrides: Optional[PrimitiveRunTimeParamValuesType],
-        run_time_param_file_overrides: Optional[CSVRunTimeParamFilesType],
+        run_time_param_file_overrides: Optional[CSVRuntimeParamPaths],
     ) -> Optional[Parameters]:
         """Extract the parameters defined in the protocol, overridden with values for the run."""
         return exec_add_parameters(

api/src/opentrons/protocol_runner/run_orchestrator.py

@@ -32,7 +32,7 @@
     DeckConfigurationType,
     RunTimeParameter,
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
 )
 from ..protocol_engine.error_recovery_policy import ErrorRecoveryPolicy
 
@@ -340,7 +340,7 @@ async def load(
         self,
         protocol_source: ProtocolSource,
         run_time_param_values: Optional[PrimitiveRunTimeParamValuesType],
-        run_time_param_files: Optional[CSVRunTimeParamFilesType],
+        run_time_param_paths: Optional[CSVRuntimeParamPaths],
         parse_mode: ParseMode,
     ) -> None:
         """Load a json/python protocol."""
@@ -356,7 +356,7 @@ async def load(
                 # doesn't conform to the new rules.
                 python_parse_mode=python_parse_mode,
                 run_time_param_values=run_time_param_values,
-                run_time_param_files=run_time_param_files,
+                run_time_param_paths=run_time_param_paths,
             )
 
     def get_is_okay_to_clear(self) -> bool:

api/src/opentrons/protocols/execution/execute.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Optional
 
-from opentrons.protocol_api import ProtocolContext
+from opentrons.protocol_api import ProtocolContext, ParameterContext
 from opentrons.protocol_api._parameters import Parameters
 from opentrons.protocols.execution.execute_python import exec_run
 from opentrons.protocols.execution.json_dispatchers import (
@@ -21,25 +21,36 @@
 def run_protocol(
     protocol: Protocol,
     context: ProtocolContext,
+    parameter_context: Optional[ParameterContext] = None,
     run_time_parameters_with_overrides: Optional[Parameters] = None,
 ) -> None:
     """Run a protocol.
 
     :param protocol: The :py:class:`.protocols.types.Protocol` to execute
     :param context: The protocol context to use.
+    :param parameter_context: The parameter context to use if running with runtime parameters.
     :param run_time_parameters_with_overrides: Run time parameters defined in the protocol,
         updated with the run's RTP override values. When we are running either simulate
         or execute, this will be None (until RTP is supported in cli commands)
     """
     if isinstance(protocol, PythonProtocol):
-        if protocol.api_level >= APIVersion(2, 0):
-            exec_run(
-                proto=protocol,
-                context=context,
-                run_time_parameters_with_overrides=run_time_parameters_with_overrides,
-            )
-        else:
-            raise RuntimeError(f"Unsupported python API version: {protocol.api_level}")
+        try:
+            if protocol.api_level >= APIVersion(2, 0):
+                exec_run(
+                    proto=protocol,
+                    context=context,
+                    run_time_parameters_with_overrides=run_time_parameters_with_overrides,
+                )
+            else:
+                raise RuntimeError(
+                    f"Unsupported python API version: {protocol.api_level}"
+                )
+        except Exception:
+            raise
+        finally:
+            # TODO(jbl 2024-08-02) this should be more tightly bound to the opening of the csv files
+            if parameter_context is not None:
+                parameter_context.close_csv_files()
     else:
         if protocol.contents["schemaVersion"] == 3:
             ins = execute_json_v3.load_pipettes_from_json(context, protocol.contents)

api/src/opentrons/protocols/execution/execute_python.py

@@ -12,7 +12,7 @@
 from opentrons.protocols.types import PythonProtocol, MalformedPythonProtocolError
 from opentrons.protocol_engine.types import (
     PrimitiveRunTimeParamValuesType,
-    CSVRunTimeParamFilesType,
+    CSVRuntimeParamPaths,
 )
 
 
@@ -71,7 +71,7 @@ def _raise_pretty_protocol_error(exception: Exception, filename: str) -> None:
 def _parse_and_set_parameters(
     parameter_context: ParameterContext,
     run_time_param_overrides: Optional[PrimitiveRunTimeParamValuesType],
-    run_time_param_file_overrides: Optional[CSVRunTimeParamFilesType],
+    run_time_param_file_overrides: Optional[CSVRuntimeParamPaths],
     new_globs: Dict[Any, Any],
     filename: str,
 ) -> Parameters:
@@ -111,7 +111,7 @@ def exec_add_parameters(
     protocol: PythonProtocol,
     parameter_context: ParameterContext,
     run_time_param_overrides: Optional[PrimitiveRunTimeParamValuesType],
-    run_time_param_file_overrides: Optional[CSVRunTimeParamFilesType],
+    run_time_param_file_overrides: Optional[CSVRuntimeParamPaths],
 ) -> Optional[Parameters]:
     """Exec the add_parameters function and get the final run time parameters with overrides."""
     new_globs: Dict[Any, Any] = {}

api/src/opentrons/protocols/parameters/types.py

@@ -4,6 +4,7 @@
 from .exceptions import RuntimeParameterRequired, ParameterValueError
 
 
+# TODO(jbl 2024-08-02) This is a public facing class and as such should be moved to the protocol_api folder
 class CSVParameter:
     def __init__(self, csv_file: Optional[TextIO]) -> None:
         self._file = csv_file

api/tests/opentrons/protocol_runner/test_protocol_runner.py

@@ -645,7 +645,7 @@ async def test_load_legacy_python(
         legacy_protocol_source,
         python_parse_mode=PythonParseMode.ALLOW_LEGACY_METADATA_AND_REQUIREMENTS,
         run_time_param_values=None,
-        run_time_param_files=None,
+        run_time_param_paths=None,
     )
 
     run_func_captor = matchers.Captor()
@@ -668,6 +668,7 @@ async def test_load_legacy_python(
         await python_protocol_executor.execute(
             protocol=legacy_protocol,
             context=protocol_context,
+            parameter_context=python_runner_subject._parameter_context,
             run_time_parameters_with_overrides=None,
         ),
     )
@@ -727,7 +728,7 @@ async def test_load_python_with_pe_papi_core(
         protocol_source,
         python_parse_mode=PythonParseMode.ALLOW_LEGACY_METADATA_AND_REQUIREMENTS,
         run_time_param_values=None,
-        run_time_param_files=None,
+        run_time_param_paths=None,
     )
 
     decoy.verify(protocol_engine.add_plugin(matchers.IsA(LegacyContextPlugin)), times=0)
@@ -790,7 +791,7 @@ async def test_load_legacy_json(
         legacy_protocol_source,
         python_parse_mode=PythonParseMode.ALLOW_LEGACY_METADATA_AND_REQUIREMENTS,
         run_time_param_values=None,
-        run_time_param_files=None,
+        run_time_param_paths=None,
     )
 
     run_func_captor = matchers.Captor()
@@ -811,6 +812,7 @@ async def test_load_legacy_json(
         await python_protocol_executor.execute(
             protocol=legacy_protocol,
             context=protocol_context,
+            parameter_context=None,
             run_time_parameters_with_overrides=None,
         ),
     )