diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 2761a6beac2..51623fb7f80 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -665,7 +665,6 @@ jobs: run: ./ci/github/unit-testing/catalog.bash install - name: typecheck run: ./ci/github/unit-testing/catalog.bash typecheck - continue-on-error: true - name: test if: always() run: ./ci/github/unit-testing/catalog.bash test diff --git a/packages/models-library/src/models_library/rpc_pagination.py b/packages/models-library/src/models_library/rpc_pagination.py index 2abaa398cde..34eeb997990 100644 --- a/packages/models-library/src/models_library/rpc_pagination.py +++ b/packages/models-library/src/models_library/rpc_pagination.py @@ -65,7 +65,7 @@ def create( total: int, limit: int, offset: int, - ): + ) -> "PageRpc": return cls( _meta=PageMetaInfoLimitOffset( total=total, count=len(chunk), limit=limit, offset=offset diff --git a/services/catalog/requirements/_test.in b/services/catalog/requirements/_test.in index e215bca4248..c6189773150 100644 --- a/services/catalog/requirements/_test.in +++ b/services/catalog/requirements/_test.in @@ -28,3 +28,5 @@ pytest-mock pytest-runner respx sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html +types-psycopg2 +types-PyYAML diff --git a/services/catalog/requirements/_test.txt b/services/catalog/requirements/_test.txt index bf6b992d58a..1c116c89286 100644 --- a/services/catalog/requirements/_test.txt +++ b/services/catalog/requirements/_test.txt @@ -185,6 +185,10 @@ tomli==2.0.1 # coverage # mypy # pytest +types-psycopg2==2.9.21.20240417 + # via -r requirements/_test.in +types-pyyaml==6.0.12.20240724 + # via -r requirements/_test.in typing-extensions==4.10.0 # via # -c requirements/_base.txt diff --git a/services/catalog/setup.cfg b/services/catalog/setup.cfg index 32bc36fdac7..144dbc1a4b9 100644 --- a/services/catalog/setup.cfg +++ b/services/catalog/setup.cfg @@ -9,5 +9,10 @@ commit_args = --no-verify [tool:pytest] asyncio_mode = auto -markers = +markers = testit: "marks test to run during development" + +[mypy] +plugins = + pydantic.mypy + sqlalchemy.ext.mypy.plugin diff --git a/services/catalog/src/simcore_service_catalog/_constants.py b/services/catalog/src/simcore_service_catalog/_constants.py index 0f3a8f3aff8..bc500c24e18 100644 --- a/services/catalog/src/simcore_service_catalog/_constants.py +++ b/services/catalog/src/simcore_service_catalog/_constants.py @@ -1,9 +1,9 @@ -from typing import Final +from typing import Any, Final # These are equivalent to pydantic export models but for responses # SEE https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict # SEE https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter -RESPONSE_MODEL_POLICY: Final[dict[str, bool]] = { +RESPONSE_MODEL_POLICY: Final[dict[str, Any]] = { "response_model_by_alias": True, "response_model_exclude_unset": True, "response_model_exclude_defaults": False, diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index 7d55dab5f59..a5d0eba83c2 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -1,6 +1,6 @@ import logging from dataclasses import dataclass -from typing import Annotated +from typing import Annotated, cast from fastapi import Depends, FastAPI, Header, HTTPException, status from models_library.api_schemas_catalog.services_specifications import ( @@ -92,10 +92,13 @@ async def get_service_from_manifest( Retrieves service metadata from the docker registry via the director """ try: - return await manifest.get_service( - key=service_key, - version=service_version, - director_client=director_client, + return cast( + ServiceMetaDataPublished, + await manifest.get_service( + key=service_key, + version=service_version, + director_client=director_client, + ), ) except ValidationError as exc: diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index 5c98e535734..68a9f6490ba 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -5,7 +5,7 @@ import urllib.parse from typing import Annotated, Any, TypeAlias, cast -from aiocache import cached +from aiocache import cached # type: ignore[import-untyped] from fastapi import APIRouter, Depends, Header, HTTPException, status from models_library.api_schemas_catalog.services import ServiceGet, ServiceUpdate from models_library.services import ServiceKey, ServiceType, ServiceVersion @@ -335,6 +335,7 @@ async def update_service( if updated_service.access_rights: # start by updating/inserting new entries + assert x_simcore_products_name # nosec new_access_rights = [ ServiceAccessRightsAtDB( key=service_key, @@ -366,6 +367,7 @@ async def update_service( await services_repo.delete_service_access_rights(deleted_access_rights) # now return the service + assert x_simcore_products_name # nosec return await get_service( user_id=user_id, service_in_manifest=await get_service_from_manifest( diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py index 84797e4832b..adc7fd338a9 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py @@ -30,11 +30,11 @@ async def list_service_ports( ports: list[ServicePortGet] = [] if service.inputs: - for name, port in service.inputs.items(): - ports.append(ServicePortGet.from_service_io("input", name, port)) + for name, input_port in service.inputs.items(): + ports.append(ServicePortGet.from_service_io("input", name, input_port)) if service.outputs: - for name, port in service.outputs.items(): - ports.append(ServicePortGet.from_service_io("output", name, port)) + for name, output_port in service.outputs.items(): + ports.append(ServicePortGet.from_service_io("output", name, output_port)) return ports diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index 88e2ab4d958..701c4b41f3d 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -197,7 +197,7 @@ async def get_service_resources( ) service_spec: ComposeSpecLabelDict | None = parse_raw_as( - ComposeSpecLabelDict | None, + ComposeSpecLabelDict | None, # type: ignore[arg-type] service_labels.get(SIMCORE_SERVICE_COMPOSE_SPEC_LABEL, "null"), ) _logger.debug("received %s", f"{service_spec=}") diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index d771326e680..e44a6d01140 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -1,5 +1,6 @@ import functools import logging +from typing import cast from fastapi import FastAPI from models_library.api_schemas_catalog.services import ( @@ -75,11 +76,14 @@ async def list_services_paginated( assert len(items) <= total_count # nosec assert len(items) <= limit # nosec - return PageRpcServicesGetV2.create( - items, - total=total_count, - limit=limit, - offset=offset, + return cast( + PageRpcServicesGetV2, + PageRpcServicesGetV2.create( + items, + total=total_count, + limit=limit, + offset=offset, + ), ) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index 89079b7561d..fa1c4cbb659 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -141,8 +141,8 @@ async def _ensure_published_templates_accessible( missing_services = published_services - available_services missing_services_access_rights = [ ServiceAccessRightsAtDB( - key=service[0], - version=service[1], + key=ServiceKey(service[0]), + version=ServiceVersion(service[1]), gid=everyone_gid, execute_access=True, product_name=default_product_name, diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py b/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py index 8cc2fc1c6f5..17512b5e8d5 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py @@ -1,3 +1,5 @@ +from typing import Any + import sqlalchemy as sa from models_library.products import ProductName from models_library.services_types import ServiceKey, ServiceVersion @@ -26,13 +28,13 @@ def list_services_stmt( ) -> Select: stmt = sa.select(services_meta_data) if gids or execute_access or write_access: - conditions = [] + conditions: list[Any] = [] # access rights logic_operator = and_ if combine_access_with_and else or_ default = bool(combine_access_with_and) - access_query_part = logic_operator( + access_query_part = logic_operator( # type: ignore[type-var] services_access_rights.c.execute_access if execute_access else default, services_access_rights.c.write_access if write_access else default, ) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index b4138eb8ed7..93f53d6bacb 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -2,7 +2,7 @@ import logging from collections import defaultdict from collections.abc import Iterable -from typing import Any, cast +from typing import Any import packaging.version import sqlalchemy as sa @@ -140,7 +140,7 @@ async def list_service_releases( # Now sort naturally from latest first: (This is lame, the sorting should be done in the db) def _by_version(x: ServiceMetaDataAtDB) -> packaging.version.Version: - return cast(packaging.version.Version, packaging.version.parse(x.version)) + return packaging.version.parse(x.version) return sorted(releases, key=_by_version, reverse=True) @@ -163,7 +163,7 @@ async def get_latest_release(self, key: str) -> ServiceMetaDataAtDB | None: result = await conn.execute(query) row = result.first() if row: - return cast(ServiceMetaDataAtDB, ServiceMetaDataAtDB.from_orm(row)) + return ServiceMetaDataAtDB.from_orm(row) return None # mypy async def get_service( @@ -207,7 +207,7 @@ async def get_service( result = await conn.execute(query) row = result.first() if row: - return cast(ServiceMetaDataAtDB, ServiceMetaDataAtDB.from_orm(row)) + return ServiceMetaDataAtDB.from_orm(row) return None # mypy async def create_or_update_service( @@ -233,9 +233,7 @@ async def create_or_update_service( ) row = result.first() assert row # nosec - created_service = cast( - ServiceMetaDataAtDB, ServiceMetaDataAtDB.from_orm(row) - ) + created_service = ServiceMetaDataAtDB.from_orm(row) for access_rights in new_service_access_rights: insert_stmt = pg_insert(services_access_rights).values( @@ -268,7 +266,7 @@ async def update_service( result = await conn.execute(stmt_update) row = result.first() assert row # nosec - return cast(ServiceMetaDataAtDB, ServiceMetaDataAtDB.from_orm(row)) + return ServiceMetaDataAtDB.from_orm(row) async def can_get_service( self, diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py index f76edb1ed6b..a28839dad42 100644 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py +++ b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from typing import Awaitable from fastapi import HTTPException from fastapi.encoders import jsonable_encoder @@ -14,7 +15,7 @@ async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse: def make_http_error_handler_for_exception( status_code: int, exception_cls: type[BaseException] -) -> Callable[[Request, type[BaseException]], JSONResponse]: +) -> Callable[[Request, type[BaseException]], Awaitable[JSONResponse]]: """ Produces a handler for BaseException-type exceptions which converts them into an error JSON response with a given status code diff --git a/services/catalog/src/simcore_service_catalog/services/access_rights.py b/services/catalog/src/simcore_service_catalog/services/access_rights.py index 73ee7f71f7d..8e1f485b0d7 100644 --- a/services/catalog/src/simcore_service_catalog/services/access_rights.py +++ b/services/catalog/src/simcore_service_catalog/services/access_rights.py @@ -1,6 +1,7 @@ """ Services Access Rights policies """ + import logging import operator from collections.abc import Callable @@ -11,6 +12,7 @@ import arrow from fastapi import FastAPI from models_library.services import ServiceMetaDataPublished +from models_library.services_types import ServiceKey, ServiceVersion from packaging.version import Version from pydantic.types import PositiveInt from sqlalchemy.ext.asyncio import AsyncEngine @@ -184,8 +186,8 @@ def _get_flags(access: ServiceAccessRightsAtDB) -> dict[str, bool]: reduced_access_rights: list[ServiceAccessRightsAtDB] = [ ServiceAccessRightsAtDB( - key=f"{target[0]}", - version=f"{target[1]}", + key=ServiceKey(f"{target[0]}"), + version=ServiceVersion(f"{target[1]}"), gid=int(target[2]), product_name=f"{target[3]}", **access_flags_map[target], diff --git a/services/catalog/src/simcore_service_catalog/services/compatibility.py b/services/catalog/src/simcore_service_catalog/services/compatibility.py index 9514c166b3e..1e9ea2b9a48 100644 --- a/services/catalog/src/simcore_service_catalog/services/compatibility.py +++ b/services/catalog/src/simcore_service_catalog/services/compatibility.py @@ -8,6 +8,7 @@ from models_library.users import UserID from packaging.specifiers import SpecifierSet from packaging.version import Version +from pydantic import parse_obj_as from simcore_service_catalog.utils.versioning import as_version from ..db.repositories.services import ServicesRepository @@ -78,12 +79,12 @@ async def _evaluate_custom_compatibility( return Compatibility( can_update_to=CompatibleService( key=other_service_key, - version=f"{latest_version}", + version=parse_obj_as(ServiceVersion, f"{latest_version}"), ) ) return Compatibility( can_update_to=CompatibleService( - version=f"{latest_version}", + version=parse_obj_as(ServiceVersion, f"{latest_version}"), ) ) @@ -95,9 +96,9 @@ async def evaluate_service_compatibility_map( product_name: ProductName, user_id: UserID, service_release_history: list[ReleaseFromDB], -) -> dict[ServiceVersion, Compatibility]: +) -> dict[ServiceVersion, Compatibility | None]: released_versions = _convert_to_versions(service_release_history) - result = {} + result: dict[ServiceVersion, Compatibility | None] = {} for release in service_release_history: compatibility = None @@ -108,16 +109,17 @@ async def evaluate_service_compatibility_map( repo=repo, target_version=release.version, released_versions=released_versions, - compatibility_policy=release.compatibility_policy, + compatibility_policy={**release.compatibility_policy}, ) elif latest_version := _get_latest_compatible_version( release.version, released_versions, ): compatibility = Compatibility( - can_update_to=CompatibleService(version=f"{latest_version}") + can_update_to=CompatibleService( + version=parse_obj_as(ServiceVersion, f"{latest_version}") + ) ) - result[release.version] = compatibility return result diff --git a/services/catalog/src/simcore_service_catalog/services/director.py b/services/catalog/src/simcore_service_catalog/services/director.py index b8de738b19a..135e2ab4b1f 100644 --- a/services/catalog/src/simcore_service_catalog/services/director.py +++ b/services/catalog/src/simcore_service_catalog/services/director.py @@ -24,7 +24,7 @@ MINUTE = 60 -_director_startup_retry_policy = { +_director_startup_retry_policy: dict[str, Any] = { # Random service startup order in swarm. # wait_random prevents saturating other services while startup # diff --git a/services/catalog/src/simcore_service_catalog/services/function_services.py b/services/catalog/src/simcore_service_catalog/services/function_services.py index e56ef218bdc..006da49d413 100644 --- a/services/catalog/src/simcore_service_catalog/services/function_services.py +++ b/services/catalog/src/simcore_service_catalog/services/function_services.py @@ -1,4 +1,5 @@ -from typing import Any, cast +# mypy: disable-error-code=truthy-function +from typing import Any from fastapi import status from fastapi.applications import FastAPI @@ -13,7 +14,7 @@ def _as_dict(model_instance: ServiceMetaDataPublished) -> dict[str, Any]: - return cast(dict[str, Any], model_instance.dict(by_alias=True, exclude_unset=True)) + return model_instance.dict(by_alias=True, exclude_unset=True) def get_function_service(key, version) -> ServiceMetaDataPublished: diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py index 59149111780..aa6caf52618 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -27,7 +27,7 @@ import logging from typing import Any, TypeAlias, cast -from aiocache import cached +from aiocache import cached # type: ignore[import-untyped] from models_library.function_services_catalog.api import iter_service_docker_data from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index dfccc672344..792fa03f912 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -5,6 +5,7 @@ ServiceGroupAccessRightsV2, ServiceUpdate, ) +from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName from models_library.rest_pagination import PageLimitInt from models_library.services_enums import ServiceType @@ -12,7 +13,7 @@ from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import NonNegativeInt +from pydantic import HttpUrl, NonNegativeInt, parse_obj_as from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( CatalogForbiddenError, CatalogItemNotFoundError, @@ -44,7 +45,7 @@ def _db_to_api_model( service_db: ServiceWithHistoryFromDB, access_rights_db: list[ServiceAccessRightsAtDB], service_manifest: ServiceMetaDataPublished, - compatibility_map: dict[ServiceKey, Compatibility] | None = None, + compatibility_map: dict[ServiceVersion, Compatibility | None] | None = None, ) -> ServiceGetV2: compatibility_map = compatibility_map or {} assert ( # nosec @@ -55,13 +56,21 @@ def _db_to_api_model( key=service_db.key, version=service_db.version, name=service_db.name, - thumbnail=service_db.thumbnail or None, + thumbnail=( + parse_obj_as(HttpUrl, service_db.thumbnail) + if service_db.thumbnail + else None + ), description=service_db.description, version_display=service_db.version_display, type=service_manifest.service_type, contact=service_manifest.contact, authors=service_manifest.authors, - owner=service_db.owner_email or None, + owner=( + LowerCaseEmailStr(service_db.owner_email) + if service_db.owner_email + else None + ), inputs=service_manifest.inputs or {}, outputs=service_manifest.outputs or {}, boot_options=service_manifest.boot_options, diff --git a/services/catalog/src/simcore_service_catalog/utils/versioning.py b/services/catalog/src/simcore_service_catalog/utils/versioning.py index cc08bfec969..d577b81bd75 100644 --- a/services/catalog/src/simcore_service_catalog/utils/versioning.py +++ b/services/catalog/src/simcore_service_catalog/utils/versioning.py @@ -18,4 +18,4 @@ def is_patch_release(version: _VersionOrStr, reference: _VersionOrStr) -> bool: """Returns True if version is a patch release from reference""" v: Version = as_version(version) r: Version = as_version(reference) - return v.major == r.major and v.minor == r.minor and r.micro < v.micro # type: ignore + return v.major == r.major and v.minor == r.minor and r.micro < v.micro