♻️Maintenance: mypy catalog (ITISFoundation#6121)

.github/workflows/ci-testing-deploy.yml

@@ -665,7 +665,6 @@ jobs:
         run: ./ci/github/unit-testing/catalog.bash install
       - name: typecheck
         run: ./ci/github/unit-testing/catalog.bash typecheck
-        continue-on-error: true
       - name: test
         if: always()
         run: ./ci/github/unit-testing/catalog.bash test

packages/models-library/src/models_library/rpc_pagination.py

@@ -65,7 +65,7 @@ def create(
         total: int,
         limit: int,
         offset: int,
-    ):
+    ) -> "PageRpc":
         return cls(
             _meta=PageMetaInfoLimitOffset(
                 total=total, count=len(chunk), limit=limit, offset=offset

services/catalog/requirements/_test.in

@@ -28,3 +28,5 @@ pytest-mock
 pytest-runner
 respx
 sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html
+types-psycopg2
+types-PyYAML

services/catalog/requirements/_test.txt

@@ -185,6 +185,10 @@ tomli==2.0.1
     #   coverage
     #   mypy
     #   pytest
+types-psycopg2==2.9.21.20240417
+    # via -r requirements/_test.in
+types-pyyaml==6.0.12.20240724
+    # via -r requirements/_test.in
 typing-extensions==4.10.0
     # via
     #   -c requirements/_base.txt

services/catalog/setup.cfg

@@ -9,5 +9,10 @@ commit_args = --no-verify
 
 [tool:pytest]
 asyncio_mode = auto
-markers = 
+markers =
 	testit: "marks test to run during development"
+
+[mypy]
+plugins =
+	pydantic.mypy
+	sqlalchemy.ext.mypy.plugin

services/catalog/src/simcore_service_catalog/_constants.py

@@ -1,9 +1,9 @@
-from typing import Final
+from typing import Any, Final
 
 # These are equivalent to pydantic export models but for responses
 # SEE https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict
 # SEE https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter
-RESPONSE_MODEL_POLICY: Final[dict[str, bool]] = {
+RESPONSE_MODEL_POLICY: Final[dict[str, Any]] = {
     "response_model_by_alias": True,
     "response_model_exclude_unset": True,
     "response_model_exclude_defaults": False,

services/catalog/src/simcore_service_catalog/api/dependencies/services.py

@@ -1,6 +1,6 @@
 import logging
 from dataclasses import dataclass
-from typing import Annotated
+from typing import Annotated, cast
 
 from fastapi import Depends, FastAPI, Header, HTTPException, status
 from models_library.api_schemas_catalog.services_specifications import (
@@ -92,10 +92,13 @@ async def get_service_from_manifest(
     Retrieves service metadata from the docker registry via the director
     """
     try:
-        return await manifest.get_service(
-            key=service_key,
-            version=service_version,
-            director_client=director_client,
+        return cast(
+            ServiceMetaDataPublished,
+            await manifest.get_service(
+                key=service_key,
+                version=service_version,
+                director_client=director_client,
+            ),
         )
 
     except ValidationError as exc:

services/catalog/src/simcore_service_catalog/api/rest/_services.py

@@ -5,7 +5,7 @@
 import urllib.parse
 from typing import Annotated, Any, TypeAlias, cast
 
-from aiocache import cached
+from aiocache import cached  # type: ignore[import-untyped]
 from fastapi import APIRouter, Depends, Header, HTTPException, status
 from models_library.api_schemas_catalog.services import ServiceGet, ServiceUpdate
 from models_library.services import ServiceKey, ServiceType, ServiceVersion
@@ -335,6 +335,7 @@ async def update_service(
 
     if updated_service.access_rights:
         # start by updating/inserting new entries
+        assert x_simcore_products_name  # nosec
         new_access_rights = [
             ServiceAccessRightsAtDB(
                 key=service_key,
@@ -366,6 +367,7 @@ async def update_service(
         await services_repo.delete_service_access_rights(deleted_access_rights)
 
     # now return the service
+    assert x_simcore_products_name  # nosec
     return await get_service(
         user_id=user_id,
         service_in_manifest=await get_service_from_manifest(

services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py

@@ -30,11 +30,11 @@ async def list_service_ports(
     ports: list[ServicePortGet] = []
 
     if service.inputs:
-        for name, port in service.inputs.items():
-            ports.append(ServicePortGet.from_service_io("input", name, port))
+        for name, input_port in service.inputs.items():
+            ports.append(ServicePortGet.from_service_io("input", name, input_port))
 
     if service.outputs:
-        for name, port in service.outputs.items():
-            ports.append(ServicePortGet.from_service_io("output", name, port))
+        for name, output_port in service.outputs.items():
+            ports.append(ServicePortGet.from_service_io("output", name, output_port))
 
     return ports

services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py

@@ -197,7 +197,7 @@ async def get_service_resources(
         )
 
     service_spec: ComposeSpecLabelDict | None = parse_raw_as(
-        ComposeSpecLabelDict | None,
+        ComposeSpecLabelDict | None,  # type: ignore[arg-type]
         service_labels.get(SIMCORE_SERVICE_COMPOSE_SPEC_LABEL, "null"),
     )
     _logger.debug("received %s", f"{service_spec=}")

services/catalog/src/simcore_service_catalog/api/rpc/_services.py

@@ -1,5 +1,6 @@
 import functools
 import logging
+from typing import cast
 
 from fastapi import FastAPI
 from models_library.api_schemas_catalog.services import (
@@ -75,11 +76,14 @@ async def list_services_paginated(
     assert len(items) <= total_count  # nosec
     assert len(items) <= limit  # nosec
 
-    return PageRpcServicesGetV2.create(
-        items,
-        total=total_count,
-        limit=limit,
-        offset=offset,
+    return cast(
+        PageRpcServicesGetV2,
+        PageRpcServicesGetV2.create(
+            items,
+            total=total_count,
+            limit=limit,
+            offset=offset,
+        ),
     )
 
 

services/catalog/src/simcore_service_catalog/core/background_tasks.py

@@ -141,8 +141,8 @@ async def _ensure_published_templates_accessible(
     missing_services = published_services - available_services
     missing_services_access_rights = [
         ServiceAccessRightsAtDB(
-            key=service[0],
-            version=service[1],
+            key=ServiceKey(service[0]),
+            version=ServiceVersion(service[1]),
             gid=everyone_gid,
             execute_access=True,
             product_name=default_product_name,

services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 import sqlalchemy as sa
 from models_library.products import ProductName
 from models_library.services_types import ServiceKey, ServiceVersion
@@ -26,13 +28,13 @@ def list_services_stmt(
 ) -> Select:
     stmt = sa.select(services_meta_data)
     if gids or execute_access or write_access:
-        conditions = []
+        conditions: list[Any] = []
 
         # access rights
         logic_operator = and_ if combine_access_with_and else or_
         default = bool(combine_access_with_and)
 
-        access_query_part = logic_operator(
+        access_query_part = logic_operator(  # type: ignore[type-var]
             services_access_rights.c.execute_access if execute_access else default,
             services_access_rights.c.write_access if write_access else default,
         )

services/catalog/src/simcore_service_catalog/db/repositories/services.py

@@ -2,7 +2,7 @@
 import logging
 from collections import defaultdict
 from collections.abc import Iterable
-from typing import Any, cast
+from typing import Any
 
 import packaging.version
 import sqlalchemy as sa
@@ -140,7 +140,7 @@ async def list_service_releases(
 
         # Now sort naturally from latest first: (This is lame, the sorting should be done in the db)
         def _by_version(x: ServiceMetaDataAtDB) -> packaging.version.Version:
-            return cast(packaging.version.Version, packaging.version.parse(x.version))
+            return packaging.version.parse(x.version)
 
         return sorted(releases, key=_by_version, reverse=True)
 
@@ -163,7 +163,7 @@ async def get_latest_release(self, key: str) -> ServiceMetaDataAtDB | None:
             result = await conn.execute(query)
             row = result.first()
         if row:
-            return cast(ServiceMetaDataAtDB, ServiceMetaDataAtDB.from_orm(row))
+            return ServiceMetaDataAtDB.from_orm(row)
         return None  # mypy
 
     async def get_service(
@@ -207,7 +207,7 @@ async def get_service(
             result = await conn.execute(query)
             row = result.first()
         if row:
-            return cast(ServiceMetaDataAtDB, ServiceMetaDataAtDB.from_orm(row))
+            return ServiceMetaDataAtDB.from_orm(row)
         return None  # mypy
 
     async def create_or_update_service(
@@ -233,9 +233,7 @@ async def create_or_update_service(
             )
             row = result.first()
             assert row  # nosec
-            created_service = cast(
-                ServiceMetaDataAtDB, ServiceMetaDataAtDB.from_orm(row)
-            )
+            created_service = ServiceMetaDataAtDB.from_orm(row)
 
             for access_rights in new_service_access_rights:
                 insert_stmt = pg_insert(services_access_rights).values(
@@ -268,7 +266,7 @@ async def update_service(
             result = await conn.execute(stmt_update)
             row = result.first()
             assert row  # nosec
-        return cast(ServiceMetaDataAtDB, ServiceMetaDataAtDB.from_orm(row))
+        return ServiceMetaDataAtDB.from_orm(row)
 
     async def can_get_service(
         self,

services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py

@@ -1,4 +1,5 @@
 from collections.abc import Callable
+from typing import Awaitable
 
 from fastapi import HTTPException
 from fastapi.encoders import jsonable_encoder
@@ -14,7 +15,7 @@ async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse:
 
 def make_http_error_handler_for_exception(
     status_code: int, exception_cls: type[BaseException]
-) -> Callable[[Request, type[BaseException]], JSONResponse]:
+) -> Callable[[Request, type[BaseException]], Awaitable[JSONResponse]]:
     """
     Produces a handler for BaseException-type exceptions which converts them
     into an error JSON response with a given status code

services/catalog/src/simcore_service_catalog/services/access_rights.py

@@ -1,6 +1,7 @@
 """ Services Access Rights policies
 
 """
+
 import logging
 import operator
 from collections.abc import Callable
@@ -11,6 +12,7 @@
 import arrow
 from fastapi import FastAPI
 from models_library.services import ServiceMetaDataPublished
+from models_library.services_types import ServiceKey, ServiceVersion
 from packaging.version import Version
 from pydantic.types import PositiveInt
 from sqlalchemy.ext.asyncio import AsyncEngine
@@ -184,8 +186,8 @@ def _get_flags(access: ServiceAccessRightsAtDB) -> dict[str, bool]:
 
     reduced_access_rights: list[ServiceAccessRightsAtDB] = [
         ServiceAccessRightsAtDB(
-            key=f"{target[0]}",
-            version=f"{target[1]}",
+            key=ServiceKey(f"{target[0]}"),
+            version=ServiceVersion(f"{target[1]}"),
             gid=int(target[2]),
             product_name=f"{target[3]}",
             **access_flags_map[target],

services/catalog/src/simcore_service_catalog/services/compatibility.py

@@ -8,6 +8,7 @@
 from models_library.users import UserID
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
+from pydantic import parse_obj_as
 from simcore_service_catalog.utils.versioning import as_version
 
 from ..db.repositories.services import ServicesRepository
@@ -78,12 +79,12 @@ async def _evaluate_custom_compatibility(
             return Compatibility(
                 can_update_to=CompatibleService(
                     key=other_service_key,
-                    version=f"{latest_version}",
+                    version=parse_obj_as(ServiceVersion, f"{latest_version}"),
                 )
             )
         return Compatibility(
             can_update_to=CompatibleService(
-                version=f"{latest_version}",
+                version=parse_obj_as(ServiceVersion, f"{latest_version}"),
             )
         )
 
@@ -95,9 +96,9 @@ async def evaluate_service_compatibility_map(
     product_name: ProductName,
     user_id: UserID,
     service_release_history: list[ReleaseFromDB],
-) -> dict[ServiceVersion, Compatibility]:
+) -> dict[ServiceVersion, Compatibility | None]:
     released_versions = _convert_to_versions(service_release_history)
-    result = {}
+    result: dict[ServiceVersion, Compatibility | None] = {}
 
     for release in service_release_history:
         compatibility = None
@@ -108,16 +109,17 @@ async def evaluate_service_compatibility_map(
                 repo=repo,
                 target_version=release.version,
                 released_versions=released_versions,
-                compatibility_policy=release.compatibility_policy,
+                compatibility_policy={**release.compatibility_policy},
             )
         elif latest_version := _get_latest_compatible_version(
             release.version,
             released_versions,
         ):
             compatibility = Compatibility(
-                can_update_to=CompatibleService(version=f"{latest_version}")
+                can_update_to=CompatibleService(
+                    version=parse_obj_as(ServiceVersion, f"{latest_version}")
+                )
             )
-
         result[release.version] = compatibility
 
     return result

services/catalog/src/simcore_service_catalog/services/director.py

@@ -24,7 +24,7 @@
 
 MINUTE = 60
 
-_director_startup_retry_policy = {
+_director_startup_retry_policy: dict[str, Any] = {
     # Random service startup order in swarm.
     # wait_random prevents saturating other services while startup
     #

services/catalog/src/simcore_service_catalog/services/function_services.py

@@ -1,4 +1,5 @@
-from typing import Any, cast
+# mypy: disable-error-code=truthy-function
+from typing import Any
 
 from fastapi import status
 from fastapi.applications import FastAPI
@@ -13,7 +14,7 @@
 
 
 def _as_dict(model_instance: ServiceMetaDataPublished) -> dict[str, Any]:
-    return cast(dict[str, Any], model_instance.dict(by_alias=True, exclude_unset=True))
+    return model_instance.dict(by_alias=True, exclude_unset=True)
 
 
 def get_function_service(key, version) -> ServiceMetaDataPublished:

services/catalog/src/simcore_service_catalog/services/manifest.py

@@ -27,7 +27,7 @@
 import logging
 from typing import Any, TypeAlias, cast
 
-from aiocache import cached
+from aiocache import cached  # type: ignore[import-untyped]
 from models_library.function_services_catalog.api import iter_service_docker_data
 from models_library.services_metadata_published import ServiceMetaDataPublished
 from models_library.services_types import ServiceKey, ServiceVersion