chore: update python deps to breaking change breakpoints (#14420)

This is a PRs to update python dependencies to modern versions. This one stops at the major breaking change point of pydantic 2.0. While Pydantic 2.0 is apparently much much faster (it's written in rust if you can believe it) it also makes a bunch of breaking api changes and in theory provides back-compat patches. Similarly, fastapi 0.100.0 switches internally to depend on pydantic 2, with similarly theoretical back compat patches. So, we've updated to,
Dependency Version Changes

In prod, major deps (full list is essentially entirely in the robot-server pipfile, see below for why):

- fastapi 0.99.0 (from 0.68.1)
- pydantic 1.10.12 (from 1.9.2)
- uvicorn 0.27.0 (from 0.14.0)
- sqlalchemy 1.4.51 (from 1.4.32 - bigger change than you think, and breaking changes above this)

In dev tooling,

- mypy 1.8.0 (from 0.981)
- flake8 7 (from like 3 or something we never updated this)
- mock 5 (from 4)
- decoy 2 (from 1)
- pytest 7.4.4 except robot-server, which is limited by tavern to 7.3 (from various)
- tavern 2.9.1 (from 1.6)

Version Definition Changes

Some people want to use our python libraries that are published on pypi. The problem with this is that our libraries - opentrons and opentrons_shared_data have explicit version pinning in their setup.py install_requires, which makes it incredibly hard for them to coexist with other python packages that aren't comaintained by us (see #11912 , #12839 ). One way to fix this would be version ranges in the setup.py and explicit versions (that are contained within those ranges, and that match or define what's present on the robot) in the pipfiles. We couldn't do this because pipenv had problems with it.

Now, however, we've upgraded pipenv, and that strategy works! And since I was going around bumping all the deps anyway, I could figure out what the actual functional dependency version boundaries were. So as part of this, opentrons (api/setup.py) and opentrons_shared_data (shared-data/python/setup.py) now have version ranges for all of their install_requires that aren't other opentrons packages, and I'm pretty sure about those version ranges. They may be smaller than would be ideal, but they're real.

api/Pipfile

@@ -3,42 +3,48 @@ url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
+[packages]
+jsonschema = "==4.17.3"
+pydantic = "==1.10.12"
+anyio = "==3.7.1"
+opentrons-shared-data = { editable = true, path = "../shared-data/python" }
+opentrons = { editable = true, path = "." }
+opentrons-hardware = { editable = true, path = "./../hardware", extras=["FLEX"] }
+numpy = "==1.22.3"
+
 [dev-packages]
 # atomicwrites and colorama are pytest dependencies on windows,
 # spec'd here to force lockfile inclusion
 # https://github.com/pypa/pipenv/issues/4408#issuecomment-668324177
 atomicwrites = { version = "==1.4.0", markers="sys_platform=='win32'" }
 colorama = { version = "==0.4.4", markers="sys_platform=='win32'" }
-coverage = "==5.1"
-mypy = "==0.981"
+coverage = "==7.4.1"
+mypy = "==1.8.0"
 numpydoc = "==0.9.1"
-pytest = "==7.0.1"
-pytest-asyncio = "~=0.16"
-pytest-cov = "==2.10.1"
+pytest = "==7.4.4"
+pytest-asyncio = "~=0.23.0"
+pytest-cov = "==4.1.0"
 pytest-lazy-fixture = "==0.6.3"
 pytest-xdist = "~=2.5.0"
 sphinx = "==5.0.1"
 twine = "==4.0.0"
 wheel = "==0.37.0"
-typeguard = "==2.13.1"
+typeguard = "==4.1.5"
 sphinx-substitution-extensions = "==2020.9.30.0"
 sphinxext-opengraph = "==0.8.1"
 sphinx-tabs = ">=3.4.1,<4"
-mock = "~=4.0.3"
-flake8 = "~=3.9.0"
-flake8-annotations = "~=2.6.2"
-flake8-docstrings = "~=1.6.0"
-flake8-noqa = "~=1.2.1"
-decoy = "~=1.11"
+mock = "==5.1.0"
+flake8 = "==7.0.0"
+flake8-annotations = "~=3.0.1"
+flake8-docstrings = "~=1.7.0"
+flake8-noqa = "~=1.4.0"
+decoy = "==2.1.1"
 black = "==22.3.0"
-types-mock = "==4.0.1"
+types-mock = "~=5.1.0"
 types-setuptools = "==57.0.2"
-opentrons-shared-data = { editable = true, path = "../shared-data/python" }
-opentrons = { editable = true, path = "." }
-opentrons-hardware = { editable = true, path = "./../hardware", extras=["FLEX"] }
 # specify typing-extensions explicitly to force lockfile inclusion on Python >= 3.8
 typing-extensions = ">=4.0.0,<5"
 pytest-profiling = "~=1.7.0"
 # TODO(mc, 2022-03-31): upgrade sphinx, remove this subdep pin
 jinja2 = ">=2.3,<3.1"
-hypothesis = ">=6.36,<7"
+hypothesis = "==6.96.1"

api/mypy.ini

@@ -4,7 +4,7 @@ show_error_codes = True
 warn_unused_configs = True
 strict = True
 # TODO(mc, 2021-09-12): work through and remove these exclusions
-exclude = tests/opentrons/(hardware_control/test_(?!ot3).*py|hardware_control/integration/|hardware_control/emulation/|hardware_control/modules/|protocols/advanced_control/|protocols/api_support/|protocols/duration/|protocols/execution/|protocols/fixtures/|protocols/geometry/)
+exclude = tests/opentrons/(hardware_control/test_(?!(ot3|module_control)).*py|hardware_control/integration/|hardware_control/emulation/|hardware_control/modules/|protocols/advanced_control/|protocols/api_support/|protocols/duration/|protocols/execution/|protocols/fixtures/|protocols/geometry/)
 
 [pydantic-mypy]
 init_forbid_extra = True

api/setup.py

@@ -60,11 +60,11 @@ def get_version():
 INSTALL_REQUIRES = [
     f"opentrons-shared-data=={VERSION}",
     "aionotify==0.2.0",
-    "anyio==3.6.1",
-    "jsonschema==3.0.2",
-    "numpy>=1.15.1,<2",
-    "pydantic==1.9.2",
-    "pyserial==3.5",
+    "anyio>=3.6.1,<4.0.0",
+    "jsonschema>=3.0.1,<4.18.0",
+    "numpy>=1.20.0,<2",
+    "pydantic>=1.10.9,<2.0.0",
+    "pyserial>=3.5",
     "typing-extensions>=4.0.0,<5",
     "click>=8.0.0,<9",
     'importlib-metadata >= 1.0 ; python_version < "3.8"',

api/src/opentrons/calibration_storage/helpers.py

@@ -5,7 +5,7 @@
 labware calibration to its designated file location.
 """
 import json
-from typing import Any, Union, List, Dict, TYPE_CHECKING, cast
+from typing import Any, Union, List, Dict, TYPE_CHECKING, cast, Tuple
 from dataclasses import is_dataclass, asdict
 
 
@@ -18,10 +18,7 @@
     from opentrons_shared_data.pipette.dev_types import LabwareUri
 
 
-DictionaryFactoryType = Union[List, Dict]
-
-
-def dict_filter_none(data: DictionaryFactoryType) -> Dict[str, Any]:
+def dict_filter_none(data: List[Tuple[str, Any]]) -> Dict[str, Any]:
     """
     Helper function to filter out None keys from a dataclass
     before saving to file.

api/src/opentrons/calibration_storage/ot2/models/v1.py

@@ -51,7 +51,7 @@ class DeckCalibrationModel(BaseModel):
     attitude: types.AttitudeMatrix = Field(
         ..., description="Attitude matrix for deck found from calibration."
     )
-    last_modified: datetime = Field(
+    last_modified: typing.Optional[datetime] = Field(
         default=None, description="The last time this deck was calibrated."
     )
     source: types.SourceType = Field(

api/src/opentrons/drivers/asyncio/communication/async_serial.py

@@ -7,7 +7,7 @@
 from typing import Optional, AsyncGenerator, Union
 from typing_extensions import Literal
 
-from serial import Serial, serial_for_url  # type: ignore[import]
+from serial import Serial, serial_for_url  # type: ignore[import-untyped]
 
 TimeoutProperties = Union[Literal["write_timeout"], Literal["timeout"]]
 

api/src/opentrons/drivers/rpi_drivers/gpio.py

@@ -7,7 +7,7 @@
 from . import RevisionPinsError
 from .types import gpio_group, PinDir, GPIOPin
 
-import gpiod  # type: ignore[import]
+import gpiod  # type: ignore[import-not-found]
 
 """
 Raspberry Pi GPIO control module

api/src/opentrons/drivers/serial_communication.py

@@ -1,12 +1,12 @@
 from typing import List, Optional, Iterator
 
-import serial  # type: ignore[import]
+import serial  # type: ignore[import-untyped]
 from serial import Serial
-from serial.tools import list_ports  # type: ignore[import]
+from serial.tools import list_ports  # type: ignore[import-untyped]
 import contextlib
 import logging
 
-from serial.tools.list_ports_common import ListPortInfo  # type: ignore[import]
+from serial.tools.list_ports_common import ListPortInfo  # type: ignore[import-untyped]
 
 log = logging.getLogger(__name__)
 

api/src/opentrons/drivers/smoothie_drivers/driver_3_0.py

@@ -18,7 +18,7 @@
 from math import isclose
 
 from opentrons.drivers.serial_communication import get_ports_by_name
-from serial.serialutil import SerialException  # type: ignore[import]
+from serial.serialutil import SerialException  # type: ignore[import-untyped]
 
 from opentrons.drivers.smoothie_drivers.connection import SmoothieConnection
 from opentrons.drivers.smoothie_drivers.constants import (

api/src/opentrons/drivers/smoothie_drivers/simulator.py

@@ -57,11 +57,11 @@ async def update_pipette_config(
         - endstop debounce M365.2 (NOT for zprobe debounce)
         - retract from endstop distance M365.3
         """
-        pass
+        return {}
 
     @property
     def current(self) -> Dict[str, float]:
-        pass
+        return {}
 
     @property
     def speed(self) -> None:

api/src/opentrons/hardware_control/backends/controller.py

@@ -19,7 +19,7 @@
 from pathlib import Path
 
 try:
-    import aionotify  # type: ignore[import]
+    import aionotify  # type: ignore[import-untyped]
 except (OSError, ModuleNotFoundError):
     aionotify = None
 

api/src/opentrons/hardware_control/backends/ot3controller.py

@@ -54,7 +54,7 @@
 from .tip_presence_manager import TipPresenceManager
 
 try:
-    import aionotify  # type: ignore[import]
+    import aionotify  # type: ignore[import-untyped]
 except (OSError, ModuleNotFoundError):
     aionotify = None
 
@@ -653,6 +653,7 @@ async def move(
             if not self._feature_flags.stall_detection_enabled
             else False,
         )
+
         mounts_moving = [
             k
             for k in moving_axes_in_move_group(move_group)
@@ -1218,7 +1219,6 @@ async def probe(self, axis: Axis, distance: float) -> OT3AxisMap[float]:
 
     async def clean_up(self) -> None:
         """Clean up."""
-
         try:
             loop = asyncio.get_event_loop()
         except RuntimeError:
@@ -1227,6 +1227,15 @@ async def clean_up(self) -> None:
         if hasattr(self, "_event_watcher"):
             if loop.is_running() and self._event_watcher:
                 self._event_watcher.close()
+
+        messenger = getattr(self, "_messenger", None)
+        if messenger:
+            await messenger.stop()
+
+        usb_messenger = getattr(self, "_usb_messenger", None)
+        if usb_messenger:
+            await usb_messenger.stop()
+
         return None
 
     @staticmethod

api/src/opentrons/hardware_control/backends/ot3simulator.py

@@ -446,7 +446,7 @@ async def home_tip_motors(
     def _attached_to_mount(
         self, mount: OT3Mount, expected_instr: Optional[PipetteName]
     ) -> OT3AttachedInstruments:
-        init_instr = self._attached_instruments.get(mount, {"model": None, "id": None})  # type: ignore
+        init_instr = self._attached_instruments.get(mount, {"model": None, "id": None})
         if mount is OT3Mount.GRIPPER:
             return self._attached_gripper_to_mount(cast(GripperSpec, init_instr))
         return self._attached_pipette_to_mount(

api/src/opentrons/hardware_control/execution_manager.py

@@ -74,7 +74,9 @@ async def wait_for_is_running(self) -> None:
         async with self._condition:
             if self._state == ExecutionState.PAUSED:
                 await self._condition.wait()
-                if self._state == ExecutionState.CANCELLED:
+                # type-ignore needed because this is a reentrant function and narrowing cannot
+                # apply
+                if self._state == ExecutionState.CANCELLED:  # type: ignore[comparison-overlap]
                     raise ExecutionCancelledError
             elif self._state == ExecutionState.CANCELLED:
                 raise ExecutionCancelledError

api/src/opentrons/hardware_control/instruments/instrument_abc.py

@@ -12,11 +12,13 @@ class AbstractInstrument(ABC, Generic[InstrumentConfig]):
     """Defines the common methods of an instrument."""
 
     @property
+    @abstractmethod
     def model(self) -> str:
         """Return model of the instrument."""
         ...
 
     @property
+    @abstractmethod
     def config(self) -> InstrumentConfig:
         """Instrument config in dataclass format."""
         ...

api/src/opentrons/hardware_control/ot3_calibration.py

@@ -9,7 +9,7 @@
 from enum import Enum
 from math import floor, copysign
 from logging import getLogger
-from opentrons.util.linal import solve_attitude, SolvePoints
+from opentrons.util.linal import solve_attitude, SolvePoints, DoubleMatrix
 
 from .types import OT3Mount, Axis, GripperProbe
 from opentrons.types import Point
@@ -952,8 +952,10 @@ def apply_machine_transform(
     -------
     Attitude matrix with regards to machine coordinate system.
     """
-    belt_attitude_arr = np.array(belt_attitude)
-    machine_transform_arr = np.array(defaults_ot3.DEFAULT_MACHINE_TRANSFORM)
+    belt_attitude_arr: DoubleMatrix = np.array(belt_attitude)
+    machine_transform_arr: DoubleMatrix = np.array(
+        defaults_ot3.DEFAULT_MACHINE_TRANSFORM
+    )
     deck_attitude_arr = np.dot(belt_attitude_arr, machine_transform_arr)
     deck_attitude = deck_attitude_arr.round(4).tolist()
     return deck_attitude  # type: ignore[no-any-return]
@@ -991,7 +993,7 @@ def validate_attitude_deck_calibration(
     TODO(pm, 5/9/2023): As with the OT2, expand on this method,
     or create another method to diagnose bad instrument offset data
     """
-    curr_cal = np.array(deck_cal.attitude)
+    curr_cal: DoubleMatrix = np.array(deck_cal.attitude)
     row, _ = curr_cal.shape
     rank: int = np.linalg.matrix_rank(curr_cal)
     if row != rank:

api/src/opentrons/hardware_control/robot_calibration.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 from dataclasses import dataclass
 from typing import Optional, List, Any, cast
+from numpy.typing import NDArray
 
 from opentrons import config
 
@@ -49,7 +50,7 @@ def validate_attitude_deck_calibration(
     TODO(lc, 8/10/2020): As with the OT2, expand on this method, or create
     another method to diagnose bad instrument offset data
     """
-    curr_cal = np.array(deck_cal.attitude)
+    curr_cal: linal.DoubleMatrix = np.array(deck_cal.attitude)
     row, _ = curr_cal.shape
     rank: int = np.linalg.matrix_rank(curr_cal)
     if row != rank:
@@ -68,7 +69,7 @@ def validate_gantry_calibration(gantry_cal: List[List[float]]) -> DeckTransformS
     This function determines whether the gantry calibration is valid
     or not based on the following use-cases:
     """
-    curr_cal = np.array(gantry_cal)
+    curr_cal: linal.DoubleMatrix = np.array(gantry_cal)
     row, _ = curr_cal.shape
 
     rank: int = np.linalg.matrix_rank(curr_cal)
@@ -95,7 +96,7 @@ def validate_gantry_calibration(gantry_cal: List[List[float]]) -> DeckTransformS
 def migrate_affine_xy_to_attitude(
     gantry_cal: List[List[float]],
 ) -> types.AttitudeMatrix:
-    masked_transform = np.array(
+    masked_transform: NDArray[np.bool_] = np.array(
         [
             [True, True, True, False],
             [True, True, True, False],
@@ -108,7 +109,7 @@ def migrate_affine_xy_to_attitude(
     ] = np.ma.masked_array(  # type: ignore
         gantry_cal, ~masked_transform
     )
-    attitude_array = np.zeros((3, 3))
+    attitude_array: linal.DoubleMatrix = np.zeros((3, 3))
     np.put(attitude_array, [0, 1, 2], masked_array[0].compressed())
     np.put(attitude_array, [3, 4, 5], masked_array[1].compressed())
     np.put(attitude_array, 8, 1)

api/src/opentrons/hardware_control/thread_manager.py

@@ -18,6 +18,7 @@
     AsyncGenerator,
     Union,
     Type,
+    ParamSpec,
 )
 from .adapters import SynchronousAdapter
 from .modules.mod_abc import AbstractModule
@@ -34,17 +35,14 @@ class ThreadManagerException(Exception):
 
 WrappedReturn = TypeVar("WrappedReturn", contravariant=True)
 WrappedYield = TypeVar("WrappedYield", contravariant=True)
-WrappedCoro = TypeVar("WrappedCoro", bound=Callable[..., Awaitable[WrappedReturn]])
-WrappedAGenFunc = TypeVar(
-    "WrappedAGenFunc", bound=Callable[..., AsyncGenerator[WrappedYield, None]]
-)
+P = ParamSpec("P")
 
 
 async def call_coroutine_threadsafe(
     loop: asyncio.AbstractEventLoop,
-    coro: WrappedCoro,
-    *args: Sequence[Any],
-    **kwargs: Mapping[str, Any],
+    coro: Callable[P, Awaitable[WrappedReturn]],
+    *args: P.args,
+    **kwargs: P.kwargs,
 ) -> WrappedReturn:
     fut = cast(
         "asyncio.Future[WrappedReturn]",
@@ -56,9 +54,9 @@ async def call_coroutine_threadsafe(
 
 async def execute_asyncgen_threadsafe(
     loop: asyncio.AbstractEventLoop,
-    agenfunc: WrappedAGenFunc,
-    *args: Sequence[Any],
-    **kwargs: Mapping[str, Any],
+    agenfunc: Callable[P, AsyncGenerator[WrappedYield, None]],
+    *args: P.args,
+    **kwargs: P.kwargs,
 ) -> AsyncGenerator[WrappedYield, None]:
 
     # This function should bridge an async generator function between two asyncio