diff --git a/LICENSE b/LICENSE index 9d94da10..7eec235d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023 Timo Thurow +Copyright (c) 2023 Hochschule Osnabrück, LMIS AG, THGA, BO-I-T Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/README.md b/README.md index 19637948..389ea0e4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ -# AW40-hub-docker - +# AW40-HUB + +
+ +## Description +This is the prototype implementation of the AW4.0 HUB architecture and part of the [Car Repair 4.0](https://www.autowerkstatt40.org/en/) research project. The purpose of the HUB is to enable car workshops to use AI driven diagnostics, persist acquired data from cars in a database as well as to participate alongside other car workshops as well as AI model providers in an [Gaia-X](https://gaia-x.eu/) compatible Dataspace to sell data and aquire new AI models. +The name AW40 is a shortened version of the german project title "Autowerkstatt 4.0". ## Requirements - Docker v25.0 or later (run `docker --version`) @@ -10,7 +20,7 @@ If you just need to update buildx, see [this section](#updating-docker-buildx-bu ## Overview -Prototype implementation of the AW40 HUB Architecture on Docker\ +This is the prototype implementation of the AW4.0 HUB Architecture.\ Currently included services: | Service (see [docker-compose.yml](docker-compose.yml)) | Description | @@ -30,8 +40,8 @@ Currently included services: ## Usage -### Start the developement HUB -**WARNING: DO NOT RUN THE DEVELOPEMENT HUB ON PUBLIC SERVER**\ +### Start the development HUB +**WARNING: DO NOT RUN THE DEVELOPMENT HUB ON PUBLIC SERVER**\ To start the HUB in developer mode use:\ ```docker compose --env-file=dev.env --profile full up -d``` diff --git a/api/Dockerfile b/api/Dockerfile index e617becd..2cd73770 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -13,6 +13,9 @@ RUN groupadd -r api && \ # use api users home directory as workdir WORKDIR /home/api +# create directory to store asset data and chown to api user +RUN mkdir /home/api/asset-data && chown api:api /home/api/asset-data + # install minimal requirements COPY ./requirements.txt /home/api/requirements.txt RUN pip install --upgrade pip && \ diff --git a/api/api.env b/api/api.env index 32b40bc2..24bc3e26 100644 --- a/api/api.env +++ b/api/api.env @@ -1,5 +1,6 @@ API_ALLOW_ORIGINS=${API_ALLOW_ORIGINS:?error} API_KEY_DIAGNOSTICS=${API_KEY_DIAGNOSTICS:?err} +API_KEY_ASSETS=${API_KEY_ASSETS:?err} MONGO_HOST=mongo MONGO_USERNAME=${MONGO_API_USERNAME:-mongo-api-user} MONGO_PASSWORD=${MONGO_API_PASSWORD:?error} diff --git a/api/api/data_management/__init__.py b/api/api/data_management/__init__.py index 87985854..8ffea393 100644 --- a/api/api/data_management/__init__.py +++ b/api/api/data_management/__init__.py @@ -1,4 +1,11 @@ __all__ = [ + "NewAsset", + "Asset", + "AssetDefinition", + "AssetMetaData", + "NewPublication", + "Publication", + "AssetDataStatus", "NewCase", "Case", "CaseUpdate", @@ -31,6 +38,10 @@ "BaseSignalStore" ] +from .assets import ( + NewAsset, AssetDefinition, Asset, AssetMetaData, Publication, + NewPublication, AssetDataStatus +) from .case import NewCase, Case, CaseUpdate from .customer import Customer, CustomerBase, CustomerUpdate from .diagnosis import ( diff --git a/api/api/data_management/assets.py b/api/api/data_management/assets.py new file mode 100644 index 00000000..13093f58 --- /dev/null +++ b/api/api/data_management/assets.py @@ -0,0 +1,182 @@ +import json +import os +from datetime import datetime, UTC +from enum import Enum +from typing import Optional, Annotated, ClassVar, Literal +from zipfile import ZipFile + +from beanie import Document, before_event, Delete +from pydantic import BaseModel, StringConstraints, Field + +from .case import Case + + +class AssetDataStatus(str, Enum): + defined = "defined" + processing = "processing" + ready = "ready" + + +class AssetDefinition(BaseModel): + """ + Defines filter conditions that cases have to match to be included in an + asset. + """ + vin: Optional[ + Annotated[str, StringConstraints(min_length=3, max_length=9)] + ] = Field( + default=None, + description="Partial VIN used to filter cases for inclusion in the " + "asset." + ) + obd_data_dtc: Optional[ + Annotated[str, StringConstraints(min_length=5, max_length=5)] + ] = Field( + default=None, + description="DTC that has to be present in a case's OBD datasets for " + "inclusion in the asset." + ) + timeseries_data_component: Optional[str] = Field( + default=None, + description="Timeseries data component that has to be present in a " + "case's timeseries datasets for inclusion in the asset." + ) + + +class PublicationNetwork(str, Enum): + pontusxdev = "PONTUSXDEV" + pontusxtest = "PONTUSXTEST" + + +class PublicationBase(BaseModel): + network: PublicationNetwork = Field( + description="Network that an asset is available in via this " + "publication", + default=PublicationNetwork.pontusxdev + ) + license: str = "CUSTOM" + price: float = 1.0 + + +class NewPublication(PublicationBase): + """Schema for new asset publications.""" + nautilus_private_key: str = Field( + description="Key for dataspace authentication." + ) + + +class Publication(PublicationBase): + """Publication information for an asset.""" + did: str = Field( + description="Id of this publication within its network." + ) + asset_url: str = Field( + description="URL to access asset data from the network." + ) + asset_key: str = Field( + description="Publication specific key to access data via `asset_url`.", + exclude=True + ) + + +class AssetMetaData(BaseModel): + name: str + definition: AssetDefinition + description: str + timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC)) + type: Literal["dataset"] = "dataset" + author: str + + +class Asset(AssetMetaData, Document): + """DB schema and interface for assets.""" + + class Settings: + name = "assets" + + data_status: AssetDataStatus = AssetDataStatus.defined + publication: Optional[Publication] = None + + asset_data_dir_path: ClassVar[str] = "asset-data" + + @staticmethod + def _publication_case_json(case: Case) -> str: + """Convert a Case into a publication ready json string.""" + # Keep WMI+VDS from VIN and mask VIS. See + # https://de.wikipedia.org/wiki/Fahrzeug-Identifizierungsnummer#Aufbau + case.vehicle_vin = case.vehicle_vin[:9] + 8*"*" + # Exclude fields only relevant for internal data management from case + exclude = { + field: True for field in [ + "customer_id", "workshop_id", "diagnosis_id", + "timeseries_data_added", "obd_data_added", "symptoms_added", + "status" + ] + } + # Exclude fields only relevant for internal data management from + # submodels + for data_submodel in ["timeseries_data", "obd_data", "symptoms"]: + exclude[data_submodel] = {"__all__": {"data_id"}} + + case_json = case.model_dump_json(exclude=exclude, indent=1) + return case_json + + @property + def data_file_name(self): + """Zip file name of the asset's dataset.""" + return f"{str(self.id)}.zip" + + @property + def data_file_path(self): + """Path to this asset's dataset.""" + return os.path.join( + self.asset_data_dir_path, self.data_file_name + ) + + async def process_definition(self): + """ + Process the definition of an Asset to prepare the defined data for + publication in a dataspace. + """ + self.data_status = AssetDataStatus.processing + await self.save() + # Find all cases matching the definition + cases = await Case.find_in_hub( + vin=self.definition.vin, + obd_data_dtc=self.definition.obd_data_dtc, + timeseries_data_component=self.definition.timeseries_data_component + ) + # Create a new zip archive for this asset + with ZipFile(self.data_file_path, "x") as archive: + archive.mkdir("cases") + archive.mkdir("signals") + for case in cases: + case_id = str(case.id) + case_json = self._publication_case_json(case) + archive.writestr( + f"cases/{case_id}.json", data=case_json + ) + for tsd in case.timeseries_data: + signal_id = str(tsd.signal_id) + signal = await tsd.get_signal() + archive.writestr( + f"signals/{signal_id}.json", data=json.dumps(signal) + ) + + self.data_status = AssetDataStatus.ready + await self.save() + + @before_event(Delete) + def _delete_asset_data(self): + """Remove associated data when asset is deleted.""" + # If there is an archive file associated with this asset, delete it. + if os.path.exists(self.data_file_path): + os.remove(self.data_file_path) + + +class NewAsset(BaseModel): + """Schema for new asset added via the api.""" + name: str + definition: Optional[AssetDefinition] = AssetDefinition() + description: str + author: str diff --git a/api/api/data_management/case.py b/api/api/data_management/case.py index deaee1c7..4509bb76 100644 --- a/api/api/data_management/case.py +++ b/api/api/data_management/case.py @@ -126,19 +126,55 @@ async def find_in_hub( cls, customer_id: Optional[str] = None, vin: Optional[str] = None, - workshop_id: Optional[str] = None + workshop_id: Optional[str] = None, + obd_data_dtc: Optional[str] = None, + timeseries_data_component: Optional[str] = None ) -> List[Self]: """ - Get list of all cases filtered by customer_id, vehicle_vin and - workshop_id. + Get list of all cases with optional filtering by customer_id, + vehicle_vin, workshop_id or obd_data dtc. + + Parameters + ---------- + customer_id + Customer Id to search for. Only cases associated with the specified + customer are returned + vin + (Partial) VIN to search for. The specified parameter value is + matched against the beginning of the stored vins. + This allows partial vin specification e.g. to search for cases with + vehicles by a specific manufacturer. + workshop_id + Workshop Id to search for. Only cases from the specified workshop + are returned. + obd_data_dtc + DTC to search for. Only cases with at least one occurrence of the + specified dtc in any of the OBD datasets are returned. + timeseries_data_component + Timeseries data component to search for. Only cases that contain at + least one timeseries dataset for the specified component are + returned. + + Returns + ------- + List of cases matching the specified search criteria. """ filter = {} if customer_id is not None: filter["customer_id"] = PydanticObjectId(customer_id) if vin is not None: - filter["vehicle_vin"] = vin + # VIN is matched against beginning of stored vins + filter["vehicle_vin"] = {"$regex": f"^{vin}"} if workshop_id is not None: filter["workshop_id"] = workshop_id + if obd_data_dtc is not None: + # Only return cases that contain the specified dtc in any + # of the obd datasets + filter["obd_data.dtcs"] = obd_data_dtc + if timeseries_data_component is not None: + # Only return cases that contain a timeseries dataset with the + # specified component + filter["timeseries_data.component"] = timeseries_data_component cases = await cls.find(filter).to_list() return cases diff --git a/api/api/dataspace_management/__init__.py b/api/api/dataspace_management/__init__.py new file mode 100644 index 00000000..25e7e08c --- /dev/null +++ b/api/api/dataspace_management/__init__.py @@ -0,0 +1,5 @@ +__all__ = [ + "Nautilus" +] + +from .nautilus import Nautilus diff --git a/api/api/dataspace_management/nautilus.py b/api/api/dataspace_management/nautilus.py new file mode 100644 index 00000000..a43fc7b2 --- /dev/null +++ b/api/api/dataspace_management/nautilus.py @@ -0,0 +1,105 @@ +from typing import Optional, Tuple + +import httpx + +from ..data_management import Asset, Publication + + +class Nautilus: + _url: Optional[str] = None + _timeout: Optional[int] = None # Timeout for external requests to nautilus + _api_key_assets: Optional[str] = None + + def __init__(self): + if not self._url: + raise AttributeError("No Nautilus connection configured.") + + @classmethod + def configure(cls, url: str, timeout: int, api_key_assets: str): + """Configure the nautilus connection details.""" + cls._url = url + cls._timeout = timeout + cls._api_key_assets = api_key_assets + + @property + def _publication_url(self): + return "/".join([self._url, "publish"]) + + @property + def _revocation_url(self): + return "/".join([self._url, "revoke"]) + + async def _post_request( + self, + url: str, + headers: dict, + json_payload: Optional[dict] = None + ) -> Tuple[Optional[httpx.Response], str]: + """ + Helper method to perform a POST request with standard error handling. + """ + try: + response = await httpx.AsyncClient().post( + url, json=json_payload, headers=headers, timeout=self._timeout + ) + response.raise_for_status() + return response, "success" + except httpx.TimeoutException: + return None, "Connection timeout." + except httpx.HTTPStatusError as e: + return None, e.response.text + + async def publish_access_dataset( + self, asset: Asset, nautilus_private_key: str + ) -> Tuple[Optional[str], str]: + """ + Publish an asset to Nautilus. + """ + # Set up request payload + payload = { + "service_descr": { + "url": asset.publication.asset_url, + "api_key": self._api_key_assets, + "data_key": asset.publication.asset_key + }, + "asset_descr": { + **asset.model_dump( + include={"name", "type", "description", "author"} + ), + "license": asset.publication.license, + "price": { + "value": asset.publication.price, + "currency": "FIXED_EUROE" + } + } + } + # Attempt publication + response, info = await self._post_request( + url="/".join( + [self._publication_url, asset.publication.network] + ), + headers={"priv_key": nautilus_private_key}, + json_payload=payload + ) + # Publication failed. No did is returned. + if response is None: + return None, info + + # Publication successful. Did is returned. + did = response.json().get("assetdid") + return did, info + + async def revoke_publication( + self, publication: Publication, nautilus_private_key: str + ) -> Tuple[bool, str]: + """Revoke a published asset in Nautilus.""" + url = "/".join( + [self._revocation_url, publication.network, publication.did] + ) + response, info = await self._post_request( + url=url, headers={"priv_key": nautilus_private_key} + ) + if response is None: + return False, info + + return True, "success" diff --git a/api/api/main.py b/api/api/main.py index e99ceb33..1454fee5 100644 --- a/api/api/main.py +++ b/api/api/main.py @@ -6,14 +6,15 @@ from .data_management import ( Case, Vehicle, Customer, Workshop, TimeseriesMetaData, Diagnosis, - AttachmentBucket + AttachmentBucket, Asset ) from .data_management.timeseries_data import GridFSSignalStore +from .dataspace_management import Nautilus from .diagnostics_management import DiagnosticTaskManager, KnowledgeGraph from .settings import settings from .security.keycloak import Keycloak from .v1 import api_v1 -from .routers import diagnostics +from .routers import diagnostics, assets app = FastAPI() app.add_middleware( @@ -42,7 +43,7 @@ async def init_mongo(): await init_beanie( client[settings.mongo_db], document_models=[ - Case, Vehicle, Customer, Workshop, Diagnosis + Case, Vehicle, Customer, Workshop, Diagnosis, Asset ] ) @@ -83,3 +84,13 @@ def init_keycloak(): @app.on_event("startup") def set_api_keys(): diagnostics.api_key_auth.valid_key = settings.api_key_diagnostics + assets.api_key_auth.valid_key = settings.api_key_assets + + +@app.on_event("startup") +def init_nautilus(): + Nautilus.configure( + url=settings.nautilus_url, + timeout=settings.nautilus_timeout, + api_key_assets=settings.api_key_assets + ) diff --git a/api/api/routers/assets.py b/api/api/routers/assets.py new file mode 100644 index 00000000..a8df8d60 --- /dev/null +++ b/api/api/routers/assets.py @@ -0,0 +1,256 @@ +import secrets +from typing import List + +from bson import ObjectId +from bson.errors import InvalidId +from fastapi import ( + APIRouter, BackgroundTasks, Depends, HTTPException, Request, Body +) +from fastapi.responses import FileResponse, JSONResponse +from fastapi.security import APIKeyHeader + +from ..data_management import ( + NewAsset, Asset, Publication, AssetDataStatus, NewPublication +) +from ..dataspace_management import Nautilus +from ..security.token_auth import authorized_assets_access +from ..security.api_key_auth import APIKeyAuth + +api_key_auth = APIKeyAuth() + +tags_metadata = [ + { + "name": "Dataspace Assets", + "description": "Proprietary dataspace asset management." + }, + { + "name": "Public Dataspace Resources", + "description": "Access to resources shared within the dataspace." + } +] + +management_router = APIRouter( + tags=["Dataspace Assets"], + prefix="/dataspace/manage", + dependencies=[Depends(authorized_assets_access)] +) + +public_router = APIRouter( + tags=["Public Dataspace Resources"], + prefix="/dataspace/public" +) + + +@management_router.get("/assets", status_code=200, response_model=List[Asset]) +async def list_assets( +): + """Retrieve list of assets.""" + return await Asset.find().to_list() + + +@management_router.post("/assets", status_code=201, response_model=Asset) +async def add_asset( + asset: NewAsset, background_tasks: BackgroundTasks +): + """ + Add a new asset. + + Afterwards, data will be processed and packaged for publication in the + background. + """ + _asset = await Asset(**asset.model_dump()).create() + background_tasks.add_task(_asset.process_definition) + return _asset + + +async def asset_by_id(asset_id: str) -> Asset: + """ + Reusable dependency to handle retrieval of assets by ID. 404 HTTP + exception is raised in case of invalid id. + """ + # Invalid ID format causes 404 + try: + asset_oid = ObjectId(asset_id) + except InvalidId: + raise HTTPException( + status_code=404, detail="Invalid format for asset_id." + ) + # Non-existing ID causes 404 + asset = await Asset.get(asset_oid) + if asset is None: + raise HTTPException( + status_code=404, + detail=f"No asset with id '{asset_id}' found." + ) + + return asset + + +@management_router.get( + "/assets/{asset_id}", status_code=200, response_model=Asset +) +async def get_asset( + asset: Asset = Depends(asset_by_id) +): + """Get an Asset by ID.""" + return asset + + +@management_router.delete( + "/assets/{asset_id}", status_code=200, response_model=None +) +async def delete_asset( + asset: Asset = Depends(asset_by_id), + nautilus: Nautilus = Depends(Nautilus), + nautilus_private_key: str = Body(embed=True) +): + """Delete an Asset and revoke any publications.""" + if asset.publication is not None: + revocation_successful, info = await nautilus.revoke_publication( + publication=asset.publication, + nautilus_private_key=nautilus_private_key + ) + if not revocation_successful: + raise HTTPException( + status_code=500, + detail=f"Failed communication with nautilus: {info}" + ) + await asset.delete() + return None + + +@management_router.get( + "/assets/{asset_id}/data", status_code=200, response_class=FileResponse +) +async def get_asset_dataset( + asset: Asset = Depends(asset_by_id), +): + """Download the dataset of an Asset.""" + if asset.data_status != AssetDataStatus.ready: + raise HTTPException( + status_code=400, + detail="Preparation of asset data hasn't finished, yet." + ) + return FileResponse( + path=asset.data_file_path, filename=asset.data_file_name + ) + + +@management_router.post( + "/assets/{asset_id}/publication", + status_code=201, + response_model=Publication +) +async def publish_asset( + new_publication: NewPublication, + request: Request, + asset: Asset = Depends(asset_by_id), + nautilus: Nautilus = Depends(Nautilus) +): + """Publish the asset in the dataspace.""" + if asset.data_status != AssetDataStatus.ready: + raise HTTPException( + status_code=400, + detail=f"Asset cannot be published until data_status is " + f"{AssetDataStatus.ready.value}." + ) + # If asset is already published, respond with publication information and + # 200 instead of 201 to indicate that no new resource was created. + if asset.publication is not None: + return JSONResponse( + content=asset.publication.model_dump(), status_code=200 + ) + + # New publication + # The full URL for data access depends on deployment and mounting prefixes. + # Hence, split requested URL by management router prefix, keep the first + # part and append the url path of the get_published_dataset endpoint to + # make sure that the asset_url points to the appropriate location for the + # current environment. + asset_url = "".join( + [ + str(request.url).split(management_router.prefix)[0], + public_router.url_path_for( + "get_published_dataset", asset_id=asset.id + ) + ] + ) + + # Might need to fix asset_url scheme if running behind reverse proxy + x_forwarded_proto = request.headers.get("x-forwarded-proto", None) + if x_forwarded_proto is not None: + if x_forwarded_proto != asset_url[:len(x_forwarded_proto)]: + asset_url = f"{x_forwarded_proto}://{asset_url.split('://')[1]}" + + # Generate a new asset key + asset_key = secrets.token_urlsafe(32) + + # Setup Publication with undetermined did + publication = Publication( + did="undetermined", + asset_key=asset_key, + asset_url=asset_url, + **new_publication.model_dump() + ) + asset.publication = publication + await asset.save() + + # Use nautilus to trigger the publication + did, info = await nautilus.publish_access_dataset( + asset=asset, + nautilus_private_key=new_publication.nautilus_private_key + ) + if did is None: + asset.publication = None + await asset.save() + raise HTTPException( + status_code=500, + detail=f"Failed communication with nautilus: {info}" + ) + + # Store the publication did + asset.publication.did = did + await asset.save() + return publication + + +@public_router.head( + "/assets/{asset_id}/data", + status_code=200, + response_class=FileResponse +) +async def get_published_dataset_head( + asset: Asset = Depends(asset_by_id) +): + return FileResponse( + path=asset.data_file_path, filename=f"{asset.name}.zip" + ) + + +@public_router.get( + "/assets/{asset_id}/data", + status_code=200, + response_class=FileResponse, + dependencies=[Depends(api_key_auth)] +) +async def get_published_dataset( + asset_key: str = Depends(APIKeyHeader(name="data_key")), + asset: Asset = Depends(asset_by_id) +): + """Public download link for asset data.""" + publication = asset.publication + if publication is None: + raise HTTPException( + status_code=404, + detail=f"No published asset with ID '{asset.id}' found." + ) + asset_key_valid = secrets.compare_digest(publication.asset_key, asset_key) + if not asset_key_valid: + raise HTTPException( + status_code=401, + detail="Could not validate asset key.", + headers={"WWW-Authenticate": "asset_key"}, + ) + return FileResponse( + path=asset.data_file_path, filename=f"{asset.name}.zip" + ) diff --git a/api/api/routers/customers.py b/api/api/routers/customers.py index 7ddb902e..474b7fdc 100644 --- a/api/api/routers/customers.py +++ b/api/api/routers/customers.py @@ -23,7 +23,7 @@ ) -@router.get("/", status_code=200, response_model=List[Customer]) +@router.get("", status_code=200, response_model=List[Customer]) async def list_customers( response: Response, request: Request, @@ -71,7 +71,7 @@ async def list_customers( return customers -@router.post("/", status_code=201, response_model=Customer) +@router.post("", status_code=201, response_model=Customer) async def add_customer(customer: CustomerBase): """Add a new customer.""" customer = await Customer(**customer.model_dump()).create() diff --git a/api/api/routers/shared.py b/api/api/routers/shared.py index c8eb46e2..eb69033f 100644 --- a/api/api/routers/shared.py +++ b/api/api/routers/shared.py @@ -22,7 +22,6 @@ } ] - router = APIRouter( tags=["Shared"], dependencies=[Depends(authorized_shared_access)] @@ -33,14 +32,24 @@ async def list_cases( customer_id: Optional[str] = None, vin: Optional[str] = None, - workshop_id: Optional[str] = None + workshop_id: Optional[str] = None, + obd_data_dtc: Optional[str] = None, + timeseries_data_component: Optional[str] = None ) -> List[Case]: """ List all cases in Hub. Query params can be used to filter by `customer_id`, - `vin` and `workshop_id`. + (partial) `vin`, `workshop_id`, `obd_data_dtc` or + `timeseries_data_component`. + + The specified `vin` is matched against the beginning of the stored vehicle + vins. """ cases = await Case.find_in_hub( - customer_id=customer_id, vin=vin, workshop_id=workshop_id + customer_id=customer_id, + vin=vin, + workshop_id=workshop_id, + obd_data_dtc=obd_data_dtc, + timeseries_data_component=timeseries_data_component ) return cases @@ -88,6 +97,7 @@ class DatasetById: Parameterized dependency to fetch a dataset by id or raise 404 if the data_id is not existent. """ + def __init__( self, data_type: Literal["timeseries_data", "obd_data", "symptom"] ): diff --git a/api/api/routers/workshop.py b/api/api/routers/workshop.py index acf6c639..c32d632c 100644 --- a/api/api/routers/workshop.py +++ b/api/api/routers/workshop.py @@ -66,10 +66,24 @@ async def list_cases( workshop_id: str, customer_id: Optional[str] = None, - vin: Optional[str] = None + vin: Optional[str] = None, + obd_data_dtc: Optional[str] = None, + timeseries_data_component: Optional[str] = None ) -> List[Case]: + """ + List all cases in Hub. Query params can be used to filter by `customer_id`, + (partial) `vin`, `workshop_id`, `obd_data_dtc` or + `timeseries_data_component`. + + The specified `vin` is matched against the beginning of the stored vehicle + vins. + """ cases = await Case.find_in_hub( - customer_id=customer_id, vin=vin, workshop_id=workshop_id + customer_id=customer_id, + vin=vin, + workshop_id=workshop_id, + obd_data_dtc=obd_data_dtc, + timeseries_data_component=timeseries_data_component ) return cases diff --git a/api/api/security/token_auth.py b/api/api/security/token_auth.py index 5d7ba3f5..f5d14db7 100644 --- a/api/api/security/token_auth.py +++ b/api/api/security/token_auth.py @@ -12,6 +12,8 @@ REQUIRED_SHARED_ROLE = "shared" # required role for customer data management REQUIRED_CUSTOMERS_ROLE = "customers" +# required role for asset data management +REQUIRED_ASSETS_ROLE = "assets" failed_auth_exception = HTTPException( @@ -104,3 +106,14 @@ async def authorized_customers_access( """ if REQUIRED_CUSTOMERS_ROLE not in token_data.roles: raise failed_auth_exception + + +async def authorized_assets_access( + token_data: TokenData = Depends(verify_token) +): + """ + Authorize access to asset data management if the user is assigned the + respective role. + """ + if REQUIRED_ASSETS_ROLE not in token_data.roles: + raise failed_auth_exception diff --git a/api/api/settings.py b/api/api/settings.py index 05d67d48..7cae615a 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -19,8 +19,13 @@ class Settings(BaseSettings): keycloak_url: str = "http://keycloak:8080" keycloak_workshop_realm: str = "werkstatt-hub" + nautilus_url: str = "http://nautilus:3000/nautilus" + nautilus_timeout: int = 120 + api_key_diagnostics: str + api_key_assets: str + exclude_diagnostics_router: bool = False @property diff --git a/api/api/v1.py b/api/api/v1.py index 5f6bc3d5..61123b38 100644 --- a/api/api/v1.py +++ b/api/api/v1.py @@ -1,10 +1,12 @@ +import logging + from fastapi import FastAPI + from .routers import ( health, shared, workshop, diagnostics, knowledge, - customers + customers, assets ) from .settings import settings -import logging class EndpointLogFilter(logging.Filter): @@ -37,6 +39,9 @@ def filter(self, record: logging.LogRecord) -> bool: api_v1.include_router(knowledge.router, prefix="/knowledge") api_v1.include_router(workshop.router) api_v1.include_router(customers.router, prefix="/customers") +# Prefixes for the assets routers are handled in the module +api_v1.include_router(assets.management_router) +api_v1.include_router(assets.public_router) if not settings.exclude_diagnostics_router: api_v1.include_router(diagnostics.router, prefix="/diagnostics") else: diff --git a/api/tests/conftest.py b/api/tests/conftest.py index f5403edd..26a4b8e3 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -9,7 +9,8 @@ Vehicle, Customer, Workshop, - Diagnosis + Diagnosis, + Asset ) from beanie import init_beanie from bson import ObjectId @@ -47,7 +48,7 @@ async def initialized_beanie_context(motor_db): context manager to handle test setup and teardown. """ models = [ - Case, Vehicle, Customer, Workshop, Diagnosis + Case, Vehicle, Customer, Workshop, Diagnosis, Asset ] class InitializedBeanieContext: @@ -317,3 +318,8 @@ def another_rsa_public_key_pem() -> bytes: """Get a public key that does not match keys from any other fixture.""" _, public_key_pem = _create_rsa_key_pair() return public_key_pem + + +@pytest.fixture(autouse=True) +def set_asset_data_dir_path_to_temporary_test_dir(tmp_path): + Asset.asset_data_dir_path = tmp_path diff --git a/api/tests/data_management/__init__.py b/api/tests/data_management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/tests/data_management/test_assets.py b/api/tests/data_management/test_assets.py new file mode 100644 index 00000000..bfef146e --- /dev/null +++ b/api/tests/data_management/test_assets.py @@ -0,0 +1,262 @@ +import json +import os +from typing import List +from zipfile import ZipFile + +import pytest +from api.data_management import ( + Asset, AssetDefinition, AssetDataStatus, Case, NewOBDData, + NewTimeseriesData, TimeseriesMetaData, NewSymptom +) +from pydantic import ValidationError + +from .test_timeseries_data import MockSignalStore + + +@pytest.fixture +def vin(): + """ + Real world VIN from + https://de.wikipedia.org/wiki/Fahrzeug-Identifizierungsnummer + """ + return "W0L000051T2123456" + + +@pytest.fixture(autouse=True) +def set_timeseries_data_signal_store_to_mock(): + TimeseriesMetaData.signal_store = MockSignalStore() + + +class TestAssetDefinition: + + def test_default(self): + # All attributes are optional + AssetDefinition() + + @pytest.mark.parametrize("vin_len", [1, 2, *range(10, 18)]) + def test_vin_length_restriction_not_met(self, vin_len, vin): + with pytest.raises(ValidationError): + AssetDefinition(vin=vin[:vin_len]) + + @pytest.mark.parametrize("vin_len", range(3, 10)) + def test_vin_length_restriction_met(self, vin_len, vin): + AssetDefinition(vin=vin[:vin_len]) + + @pytest.mark.parametrize("dtc", ["P", "P0", "P00", "P000", "P00000"]) + def test_invalid_dtc(self, dtc): + with pytest.raises(ValidationError): + AssetDefinition(obd_data_dtc=dtc) + + def test_valid_dtc(self): + AssetDefinition(obd_data_dtc="P4242") + + +class TestAsset: + + def _check_archive_case_data(self, case: Case, archive_case_data: dict): + """ + Validates contents of a single case stored in an archive. + """ + assert archive_case_data["id"] == str(case.id) + # Confirm that the vin is correctly masked + assert ( + archive_case_data["vehicle_vin"] == + case.vehicle_vin[:9] + 8 * "*" + ) + + # Confirm that fields only relevant to internal data management + # are removed from top-level and submodels + for field in [ + "customer_id", "workshop_id", "diagnosis_id", + "timeseries_data_added", "obd_data_added", "symptoms_added" + ]: + assert field not in archive_case_data + for submodel in ["timeseries_data", "obd_data", "symptoms"]: + for submodel_entry in archive_case_data[submodel]: + assert "data_id" not in submodel_entry + + def _check_archive(self, archive: ZipFile, expected_cases: List[Case]): + """ + Validates the structure and contents of archives generated with the + process_definition_method. + """ + # Get a list of all members (files and directories) in the archive + archive_members = archive.namelist() + + # Ensure the presence of the presence of the expected "cases/" and + # "signals/" directories + assert "cases/" in archive_members + assert "signals/" in archive_members + + # Track the number of signal files in the archive + signals_in_archive = 0 + + for case in expected_cases: + # Ensure the expected case path exists in the archive + archive_case_path = f"cases/{str(case.id)}.json" + assert archive_case_path in archive_members + + # Load and validate case data stored in the archive + archive_case_data = json.loads(archive.read(archive_case_path)) + self._check_archive_case_data(case, archive_case_data) + + for tsd in archive_case_data["timeseries_data"]: + signals_in_archive += 1 + # Ensure the expected signal path exists in the archive + archive_signal_path = f"signals/{tsd['signal_id']}.json" + assert archive_signal_path in archive_members + # Ensure the signal file is valid JSON + assert json.loads(archive.read(archive_signal_path)) + + # Confirm that there is nothing but the expected members (2 directories + # + cases + signals) in the archive + assert ( + len(archive_members) == 2 + + len(expected_cases) + + signals_in_archive + ) + + @pytest.mark.parametrize( + "definition,expected_cases_idx", + [ + (AssetDefinition(), [0, 1, 2]), + (AssetDefinition(vin="W0L"), [0, 1]), + (AssetDefinition(vin="W0L1"), [0]), + (AssetDefinition(obd_data_dtc="P0001"), [0, 2]), + (AssetDefinition(timeseries_data_component="CompA"), [1, 2]), + (AssetDefinition(vin="W0L", obd_data_dtc="P0001"), [0]), + ( + AssetDefinition( + vin="W0L", timeseries_data_component="CompA" + ), + [1] + ), + ( + AssetDefinition( + vin="W0L", + timeseries_data_component="CompB", + obd_data_dtc="P0001" + ), + # No case matches the definition. Hence, archive is + # expected to be empty. + [] + ) + ] + ) + @pytest.mark.asyncio + async def test_process_definition( + self, + definition, + expected_cases_idx, + initialized_beanie_context + ): + async with initialized_beanie_context: + # Three cases with different VINs are stored in the db + cases = [] + cases.append( + Case(vehicle_vin="W0L111111T1111111", workshop_id="a") + ) + cases.append( + Case(vehicle_vin="W0L222222T2222222", workshop_id="b") + ) + cases.append( + Case(vehicle_vin="1111111T150000L0W", workshop_id="c") + ) + for case in cases: + await case.create() + + # Add an OBD Dataset to each case + await cases[0].add_obd_data(NewOBDData(dtcs=["P0001"])) + await cases[1].add_obd_data(NewOBDData(dtcs=["Q0002"])) + await cases[2].add_obd_data(NewOBDData(dtcs=["P0001"])) + + # Add timeseries data to subset of cases + await cases[1].add_timeseries_data( + NewTimeseriesData( + signal=[.0, .1], + sampling_rate=1, + duration=2, + component="CompA", + label="unknown" + ) + ) + await cases[1].add_timeseries_data( + NewTimeseriesData( + signal=[.0, .1, .2], + sampling_rate=1, + duration=3, + component="CompB", + label="unknown" + ) + ) + await cases[2].add_timeseries_data( + NewTimeseriesData( + signal=[.0, .1, .2, .3], + sampling_rate=2, + duration=2, + component="CompA", + label="unknown" + ) + ) + + # Add a symptom to one case + await cases[0].add_symptom( + NewSymptom(component="CompC", label="defect") + ) + + # Create an asset with the parametrized definition + asset = await Asset( + name="Test Asset", + description="This is an test asset.", + definition=definition, + author="test author" + ).create() + + # Process the definition + await asset.process_definition() + # Assert up-to-date data_status in db + await asset.sync() + assert asset.data_status == AssetDataStatus.ready + + # Check the zip archive generated for the parametrized definition + with ZipFile(asset.data_file_path, "r") as archive: + self._check_archive( + archive=archive, + expected_cases=[cases[i] for i in expected_cases_idx] + ) + + @pytest.mark.asyncio + async def test__delete_asset_data(self, initialized_beanie_context): + """ + Confirm automatic deletion of asset data archive upon asset deletion. + """ + async with initialized_beanie_context: + asset = await Asset( + name="Test Asset", + description="This is an test asset.", + definition=AssetDefinition(), + author="Test author" + ).create() + # Test existence of archive file after processing the definition + await asset.process_definition() + assert os.path.exists(asset.data_file_path) + # Test non-existence of archive file after deleting asset from db + await asset.delete() + assert not os.path.exists(asset.data_file_path) + + @pytest.mark.asyncio + async def test__delete_asset_data_without_file( + self, initialized_beanie_context + ): + """ + Confirm that asset deletion passes without an existing archive file. + """ + async with initialized_beanie_context: + asset = await Asset( + name="Test Asset", + description="This is an test asset.", + definition=AssetDefinition(), + author="Test author" + ).create() + # Delete before an archive file was created + await asset.delete() diff --git a/api/tests/data_management/test_case.py b/api/tests/data_management/test_case.py index dbc085cd..1454e199 100644 --- a/api/tests/data_management/test_case.py +++ b/api/tests/data_management/test_case.py @@ -15,6 +15,7 @@ SymptomUpdate, SymptomLabel ) +from bson import ObjectId from pydantic import ValidationError @@ -34,9 +35,9 @@ def case_with_diagnostic_data(new_case, timeseries_data): new_case["timeseries_data"] = timeseries_data new_case["obd_data"] = {"dtcs": ["P0001"]} new_case["symptoms"] = { - "component": "battery", - "label": "defect" - } + "component": "battery", + "label": "defect" + } return new_case @@ -160,6 +161,147 @@ async def test_find_in_hub( assert case_2_result[0].vehicle_vin == case_2_vin assert case_3_result[0].workshop_id == case_3_workshop_id + @pytest.mark.parametrize( + "query_vin,expected_vins", + [ + ("AB", ["ABC", "ABCD"]), + ("ABC", ["ABC", "ABCD"]), + ("ABCD", ["ABCD"]), + ("BC", []) + ] + ) + @pytest.mark.asyncio + async def test_find_in_hub_with_partial_vin( + self, initialized_beanie_context, query_vin, expected_vins + ): + async with initialized_beanie_context: + workshop_id = "test-workshop" + await Case(vehicle_vin="ABC", workshop_id=workshop_id).create() + await Case(vehicle_vin="ABCD", workshop_id=workshop_id).create() + await Case(vehicle_vin="ZABC", workshop_id=workshop_id).create() + + retrieved_cases = await Case.find_in_hub(vin=query_vin) + retrieved_vins = sorted([_.vehicle_vin for _ in retrieved_cases]) + assert retrieved_vins == expected_vins + + @pytest.mark.parametrize( + "query_dtc,expected_cases", + [ + ("P0001", [0, 1]), + ("Q0002", [1]), + ("Z0001", []) + ] + ) + @pytest.mark.asyncio + async def test_find_in_hub_by_obd_data_dtc( + self, initialized_beanie_context, query_dtc, expected_cases + ): + async with initialized_beanie_context: + workshop_id = "test-workshop" + vin = "test-vin" + # Three cases are put into the db + cases = [] + for _ in range(3): + case = await Case( + vehicle_vin=vin, workshop_id=workshop_id + ).create() + cases.append(case) + # OBD data with dtcs is added to two of the three cases + await cases[0].add_obd_data( + NewOBDData(dtcs=["P0001"]) + ) + await cases[1].add_obd_data( + NewOBDData(dtcs=["P0001", "Q0002"]) + ) + + retrieved_cases = await Case.find_in_hub(obd_data_dtc=query_dtc) + retrieved_case_ids = sorted([_.id for _ in retrieved_cases]) + expected_case_ids = sorted([cases[i].id for i in expected_cases]) + assert retrieved_case_ids == expected_case_ids + + @pytest.mark.parametrize("query_dtc", ["P0001", "Q0001", "Z0002"]) + @pytest.mark.asyncio + async def test_find_in_hub_by_obd_data_dtc_with_multiple_datasets( + self, initialized_beanie_context, query_dtc + ): + """Test non-standard situation with multiple obd datasets in a case.""" + async with initialized_beanie_context: + case = await Case(workshop_id="42", vehicle_vin="42").create() + await case.add_obd_data( + NewOBDData(dtcs=["P0001", "Z0002"]) + ) + await case.add_obd_data( + NewOBDData(dtcs=["Q0001", "Z0002"]) + ) + await case.add_obd_data( + NewOBDData(dtcs=[]) + ) + retrieved_cases = await Case.find_in_hub(obd_data_dtc=query_dtc) + assert [_.id for _ in retrieved_cases] == [case.id] + + @pytest.mark.parametrize( + "query_component,expected_cases", + [ + ("Comp-A", [0, 1]), + ("Comp-B", [1]), + ("Comp-C", []) + ] + ) + @pytest.mark.asyncio + async def test_find_in_hub_by_timeseries_data_component( + self, + monkeypatch, + initialized_beanie_context, + query_component, + expected_cases + ): + # Patch to_timeseries_data to avoid signal store configuration + async def mock_to_timeseries_data(self): + # Just exchange signal for signal_id without storing the signal + meta_data = self.model_dump(exclude={"signal"}) + meta_data["signal_id"] = ObjectId() + return TimeseriesData(**meta_data) + + monkeypatch.setattr( + NewTimeseriesData, "to_timeseries_data", mock_to_timeseries_data + ) + + async with initialized_beanie_context: + workshop_id = "test-workshop" + vin = "test-vin" + # Three cases are put into the db + cases = [] + for _ in range(3): + case = await Case( + vehicle_vin=vin, workshop_id=workshop_id + ).create() + cases.append(case) + + common_kwargs = { + "signal": [.42], + "sampling_rate": 1, + "duration": 1, + "label": "unknown" + } + # One dataset is added to first case + await cases[0].add_timeseries_data( + NewTimeseriesData(component="Comp-A", **common_kwargs) + ) + # Two datasets are added to second case + await cases[1].add_timeseries_data( + NewTimeseriesData(component="Comp-B", **common_kwargs) + ) + await cases[1].add_timeseries_data( + NewTimeseriesData(component="Comp-A", **common_kwargs) + ) + + retrieved_cases = await Case.find_in_hub( + timeseries_data_component=query_component + ) + retrieved_case_ids = sorted([_.id for _ in retrieved_cases]) + expected_case_ids = sorted([cases[i].id for i in expected_cases]) + assert retrieved_case_ids == expected_case_ids + @pytest.mark.asyncio async def test_data_counter_are_correctly_initilialized( self, new_case, initialized_beanie_context @@ -192,6 +334,7 @@ class MockNewTimeseriesData(NewTimeseriesData): A mock for NewTimeseriesData that does not interact with a signal store when executing to_timeseries_data. """ + async def to_timeseries_data(self): signal_id = test_signal_id meta_data = self.model_dump(exclude={"signal"}) @@ -264,10 +407,12 @@ async def test_add_symptom( case.symptoms_added = previous_adds await case.add_symptom( - NewSymptom(**{ - "component": "battery", - "label": SymptomLabel("defect") - }) + NewSymptom( + **{ + "component": "battery", + "label": SymptomLabel("defect") + } + ) ) # refetch case and assert existence of single symptom diff --git a/api/tests/routers/test_assets.py b/api/tests/routers/test_assets.py new file mode 100644 index 00000000..ec77ef7b --- /dev/null +++ b/api/tests/routers/test_assets.py @@ -0,0 +1,928 @@ +import os +from datetime import datetime, timedelta, UTC +from unittest.mock import AsyncMock +from zipfile import ZipFile + +import httpx +import pytest +from api.data_management import ( + Asset, AssetDefinition, Publication +) +from api.dataspace_management import nautilus +from api.routers import assets +from api.routers.assets import Nautilus +from api.security.keycloak import Keycloak +from bson import ObjectId +from fastapi import FastAPI +from fastapi.testclient import TestClient +from jose import jws + + +@pytest.fixture +def jwt_payload(): + return { + "iat": datetime.now(UTC).timestamp(), + "exp": (datetime.now(UTC) + timedelta(60)).timestamp(), + "preferred_username": "some-user-with-assets-access", + "realm_access": {"roles": ["assets"]} + } + + +@pytest.fixture +def signed_jwt(jwt_payload, rsa_private_key_pem: bytes): + """Create a JWT signed with private RSA key.""" + return jws.sign(jwt_payload, rsa_private_key_pem, algorithm="RS256") + + +@pytest.fixture +def app(motor_db): + app = FastAPI() + app.include_router(assets.management_router) + app.include_router(assets.public_router) + assets.api_key_auth.valid_key = "assets-key-dev" + yield app + + +@pytest.fixture +def unauthenticated_client(app): + """Unauthenticated client, e.g. no bearer token in header.""" + yield TestClient(app) + + +@pytest.fixture +def authenticated_client( + unauthenticated_client, rsa_public_key_pem, signed_jwt +): + """Turn unauthenticated client into authenticated client.""" + + # Client gets auth header with valid bearer token + client = unauthenticated_client + client.headers.update({"Authorization": f"Bearer {signed_jwt}"}) + + # Make app use public key from fixture for token validation + app = client.app + app.dependency_overrides[ + Keycloak.get_public_key_for_workshop_realm + ] = lambda: rsa_public_key_pem.decode() + + return client + + +@pytest.fixture +def base_url(): + return "http://testserver" + + +@pytest.fixture +def authenticated_async_client( + app, rsa_public_key_pem, signed_jwt, base_url +): + """ + Authenticated async client for tests that require mongodb access via + beanie. Note that for this module, this is the client authorized to + manage assets via /dataspace/manage/... + """ + + # Client with valid auth header + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url=base_url, + headers={"Authorization": f"Bearer {signed_jwt}"} + ) + + # Make app use public key from fixture for token validation + app.dependency_overrides[ + Keycloak.get_public_key_for_workshop_realm + ] = lambda: rsa_public_key_pem.decode() + + return client + + +@pytest.fixture +def n_assets_in_data_context(): + return 2 + + +@pytest.fixture +def asset_ids_in_data_context(n_assets_in_data_context): + """Valid asset_id, e.g. needs to work with PydanticObjectId""" + return [str(ObjectId()) for _ in range(n_assets_in_data_context)] + + +@pytest.fixture +def data_context( + motor_db, asset_ids_in_data_context +): + """ + Seed db with test data. + + Usage: `async with initialized_beanie_context, data_context: ...` + """ + + class DataContext: + async def __aenter__(self): + # Seed the db with a few assets + for i, a_id in enumerate(asset_ids_in_data_context): + await Asset( + id=a_id, + name=f"A{i}", + description=f"This is asset {i}.", + definition=AssetDefinition(), + author="Test author" + ).create() + + async def __aexit__(self, exc_type, exc, tb): + pass + + return DataContext() + + +@pytest.mark.asyncio +async def test_list_assets_in_empty_db( + authenticated_async_client, initialized_beanie_context +): + async with initialized_beanie_context: + response = await authenticated_async_client.get( + "/dataspace/manage/assets" + ) + assert response.status_code == 200 + assert response.json() == [] + + +@pytest.mark.asyncio +async def test_list_assets( + authenticated_async_client, + initialized_beanie_context, + data_context, + n_assets_in_data_context +): + async with initialized_beanie_context, data_context: + # Request without any additional params + response = await authenticated_async_client.get( + "/dataspace/manage/assets" + ) + # Validate response status code and data + assert response.status_code == 200 + assert len(response.json()) == n_assets_in_data_context + + +@pytest.mark.asyncio +async def test_add_asset( + authenticated_async_client, + initialized_beanie_context +): + name = "New Asset" + description = "A new asset added via the api." + async with initialized_beanie_context: + response = await authenticated_async_client.post( + "/dataspace/manage/assets", + json={ + "name": name, + "description": description, + "definition": {}, + "author": "Test author" + } + ) + assert response.status_code == 201 + # Confirm asset data in response + response_data = response.json() + assert response_data["name"] == name + assert response_data["description"] == description + assert response_data["data_status"] == "defined" + # Confirm storage in db + asset_db = await Asset.get(response_data["_id"]) + assert asset_db + assert asset_db.name == name + assert asset_db.description == description + # Confirm processing of the asset + assert asset_db.data_status == "ready" + assert os.path.exists(asset_db.data_file_path) + + +@pytest.mark.asyncio +async def test_get_asset( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + response = await authenticated_async_client.get( + f"/dataspace/manage/assets/{asset_id}" + ) + assert response.status_code == 200 + assert response.json()["_id"] == asset_id + + +@pytest.fixture +def patch_nautilus_to_fail_revocation( + authenticated_async_client, monkeypatch +): + """ + Patch Nautilus to enforce failure of any attempt to revoke a publication + in the dataspace. + """ + # Configure url to avoid failure of Nautilus constructor + Nautilus.configure( + url="http://nothing-here", + timeout=None, + api_key_assets=None + ) + + def _raise(*args, **kwargs): + raise Exception("Simulated failure during asset revocation") + + monkeypatch.setattr(Nautilus, "revoke_publication", _raise) + yield + # Clean up + Nautilus.configure(url=None, timeout=None, api_key_assets=None) + + +@pytest.mark.asyncio +async def test_delete_asset( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_fail_revocation # ... as there is no publication +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + response = await authenticated_async_client.request( + "DELETE", + f"/dataspace/manage/assets/{asset_id}", + json={"nautilus_private_key": "42"} + ) + assert response.status_code == 200 + assert response.json() is None + # Confirm deletion in db + asset_db = await Asset.get(asset_id) + assert asset_db is None + + +@pytest.fixture +def patch_nautilus_to_avoid_external_revocation_request( + authenticated_async_client, monkeypatch +): + """ + Patch Nautilus to avoid external request for asset revocation + """ + # Configure url to avoid failure of Nautilus constructor + Nautilus.configure( + url="http://nothing-here", + timeout=None, + api_key_assets=None + ) + + # Create mock for the httpx.AsyncClient.post method + mock_post = AsyncMock(spec=httpx.AsyncClient.post) + mock_post.return_value = httpx.Response( + status_code=200, + request=httpx.Request("POST", "http://nothing-here") + ) + + # Patch httpx.AsyncClient.post in the nautilus module to avoid external + # request + monkeypatch.setattr( + nautilus.httpx.AsyncClient, + "post", + mock_post + ) + + yield + + # Clean up + Nautilus.configure(url=None, timeout=None, api_key_assets=None) + + +@pytest.mark.asyncio +async def test_delete_asset_with_publication( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_avoid_external_revocation_request +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + # Get one of the assets in the data_context, process it's definition + # and add a publication + asset = await Asset.get(asset_id) + await asset.process_definition() + asset.publication = Publication( + did="some-did", + asset_key="some-key", + asset_url="http://some-url" + ) + await asset.save() + # Delete it + response = await authenticated_async_client.request( + "DELETE", + f"/dataspace/manage/assets/{asset_id}", + json={"nautilus_private_key": "42"} + ) + assert response.status_code == 200 + assert response.json() is None + # Confirm deletion in db + asset_db = await Asset.get(asset_id) + assert asset_db is None + + +@pytest.mark.asyncio +async def test_get_asset_dataset_not_ready( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + # Attempt to retrieve asset data before the asset definition was + # processed + response = await authenticated_async_client.get( + f"/dataspace/manage/assets/{asset_id}/data" + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_get_asset_dataset( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + tmp_path +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + # Process asset definition + asset = await Asset.get(asset_id) + await asset.process_definition() + # Attempt to retrieve asset data after successful processing + response = await authenticated_async_client.get( + f"/dataspace/manage/assets/{asset_id}/data" + ) + assert response.status_code == 200 + # Download the archive and validate structure + download_path = tmp_path / "download.zip" + with open(download_path, "wb") as file: + file.write(response.content) + with ZipFile(download_path, "r") as archive: + # Get a list of all members (files and directories) in the archive + archive_members = archive.namelist() + # Ensure the presence of the presence of the expected "cases/" and + # "signals/" directories + assert "cases/" in archive_members + assert "signals/" in archive_members + + +@pytest.fixture +def patch_nautilus_to_fail_publication( + authenticated_async_client, monkeypatch +): + """ + Patch Nautilus to enforce failure of any attempt to publish to the + dataspace. + """ + # Configure url to avoid failure of Nautilus constructor + Nautilus.configure( + url="http://nothing-here", + timeout=None, + api_key_assets=None + ) + + def _raise(): + raise Exception("Simulated failure during dataset publication") + + monkeypatch.setattr(Nautilus, "publish_access_dataset", _raise) + yield + # Clean up + Nautilus.configure(url=None, timeout=None, api_key_assets=None) + + +@pytest.mark.asyncio +async def test_publish_asset_not_ready( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_fail_publication +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + # Attempt to publish asset before the asset definition was + # processed + response = await authenticated_async_client.post( + f"/dataspace/manage/assets/{asset_id}/publication", + json={"nautilus_private_key": "42"} + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_publish_asset_already_published( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_fail_publication +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + # Get one of the assets in the data_context, process it's definition + # and add a publication + asset = await Asset.get(asset_id) + await asset.process_definition() + asset.publication = Publication( + did="some-did", + asset_key="some-key", + asset_url="http://some-url" + ) + await asset.save() + # Attempt to publish the asset that already has a publication + response = await authenticated_async_client.post( + f"/dataspace/manage/assets/{asset_id}/publication", + json={"nautilus_private_key": "42"} + ) + # Response should indicate success but without creation of a new resource + # via 200 status code. + assert response.status_code == 200 + # Client should receive information about the existing publication + assert response.json() == asset.publication.model_dump() + + +@pytest.fixture +def patch_nautilus_to_avoid_external_request( + authenticated_async_client, monkeypatch +): + """ + Patch Nautilus to just return a publication without first attempting any + external http requests. + """ + # Configure url to avoid failure of Nautilus constructor + Nautilus.configure( + url="http://nothing-here", + timeout=None, + api_key_assets=None + ) + + # Create mock of httpx.AsyncClient + class MockAsyncClient: + async def post(self, url, headers, timeout, json): + return httpx.Response( + status_code=201, + request=httpx.Request("post", url), + json={"assetdid": "newdid"} + ) + + # Patch httpx.AsyncClient.post in the nautilus module to avoid external + # request + monkeypatch.setattr( + nautilus.httpx, "AsyncClient", MockAsyncClient + ) + yield + # Clean up + Nautilus.configure(url=None, timeout=None, api_key_assets=None) + + +@pytest.mark.asyncio +async def test_publish_asset( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_avoid_external_request +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + # Process asset definition to allow publication + asset = await Asset.get(asset_id) + await asset.process_definition() + # Attempt to publish to dataspace + response = await authenticated_async_client.post( + f"/dataspace/manage/assets/{asset_id}/publication", + json={"nautilus_private_key": "42"} + ) + # Status code should indicate creation of new resource + assert response.status_code == 201 + # The asset db object should contain a publication including asset_key + await asset.sync() + assert asset.publication.asset_key + # Response data should include all publication information except the + # asset key + assert response.json() == asset.publication.model_dump( + exclude={"asset_key"} + ) + + +@pytest.fixture +def patch_nautilus_to_timeout_communication( + authenticated_async_client, monkeypatch +): + """ + Patch Nautilus such that external publication request times out. + """ + # Configure url to avoid failure of Nautilus constructor + Nautilus.configure( + url="http://nothing-here", + timeout=None, + api_key_assets=None + ) + + # Patch httpx.AsyncClient.post in the nautilus module to timeout + class MockAsyncClient: + async def post(self, url, headers, timeout, json): + raise httpx.TimeoutException( + "Simulated timeout during dataset publication" + ) + + monkeypatch.setattr( + nautilus.httpx, "AsyncClient", MockAsyncClient + ) + yield + # Clean up + Nautilus.configure(url=None, timeout=None, api_key_assets=None) + + +@pytest.mark.asyncio +async def test_publish_asset_with_communication_timeout( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_timeout_communication +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + # Process asset definition to allow publication + asset = await Asset.get(asset_id) + await asset.process_definition() + # Attempt to publish to dataspace + response = await authenticated_async_client.post( + f"/dataspace/manage/assets/{asset_id}/publication", + json={"nautilus_private_key": "42"} + ) + # Http exception should indicate failed communication + assert response.status_code == 500 + assert response.json()["detail"] == ("Failed communication with " + "nautilus: Connection timeout.") + + +@pytest.fixture(params=[400, 401, 500, 501]) +def patch_nautilus_to_fail_http_communication( + authenticated_async_client, monkeypatch, request +): + """ + Patch Nautilus such that external publication request fails with + non-success http status code. + """ + # Configure url to avoid failure of Nautilus constructor + Nautilus.configure( + url="http://nothing-here", + timeout=None, + api_key_assets=None + ) + + # Patch httpx.AsyncClient.post in the nautilus module to avoid external + # request and to respond with non-success http code + class MockAsyncClient: + async def post(self, url, headers, timeout, json): + return httpx.Response( + status_code=request.param, + text="Failed.", + request=httpx.Request("post", url) + ) + + monkeypatch.setattr( + nautilus.httpx, "AsyncClient", MockAsyncClient + ) + yield + # Clean up + Nautilus.configure(url=None, timeout=None, api_key_assets=None) + + +@pytest.mark.asyncio +async def test_publish_asset_with_failed_http_communication( + authenticated_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_fail_http_communication +): + asset_id = asset_ids_in_data_context[0] + async with initialized_beanie_context, data_context: + # Process asset definition to allow publication + asset = await Asset.get(asset_id) + await asset.process_definition() + # Attempt to publish to dataspace + response = await authenticated_async_client.post( + f"/dataspace/manage/assets/{asset_id}/publication", + json={"nautilus_private_key": "42"} + ) + # Http exception should indicate failed communication + assert response.status_code == 500 + assert response.json()["detail"] == ("Failed communication with " + "nautilus: Failed.") + + +@pytest.mark.parametrize( + "method,endpoint", + [ + ("get", ""), + ("delete", ""), + ("get", "/data"), + ("post", "/publication") + ] +) +@pytest.mark.asyncio +async def test_asset_not_found( + method, + endpoint, + authenticated_async_client, + initialized_beanie_context +): + # Fresh ID and no data initialization + asset_id = str(ObjectId()) + async with initialized_beanie_context: + response = await authenticated_async_client.request( + method=method, url=f"/dataspace/manage/assets/{asset_id}{endpoint}" + ) + assert response.status_code == 404 + assert response.json() == { + "detail": f"No asset with id '{asset_id}' found." + } + + +@pytest.mark.parametrize( + "route", assets.management_router.routes, ids=lambda r: r.name +) +def test_missing_bearer_token(route, unauthenticated_client): + """Endpoints should not be accessible without a bearer token.""" + assert len(route.methods) == 1, "Test assumes one method per route." + method = next(iter(route.methods)) + response = unauthenticated_client.request(method=method, url=route.path) + assert response.status_code == 403 + assert response.json() == {"detail": "Not authenticated"} + + +@pytest.fixture +def jwt_payload_with_unauthorized_role(jwt_payload): + jwt_payload["realm_access"]["roles"] = ["workshop", "not assets"] + return jwt_payload + + +@pytest.fixture +def signed_jwt_with_unauthorized_role( + jwt_payload_with_unauthorized_role, rsa_private_key_pem: bytes +): + return jws.sign( + jwt_payload_with_unauthorized_role, + rsa_private_key_pem, algorithm="RS256" + ) + + +@pytest.mark.parametrize( + "route", assets.management_router.routes, ids=lambda r: r.name +) +def test_unauthorized_user( + route, authenticated_client, signed_jwt_with_unauthorized_role +): + """ + Endpoints should not be accessible, if the user role encoded in the + token does not indicate assets access. + """ + assert len(route.methods) == 1, "Test assumes one method per route." + method = next(iter(route.methods)) + authenticated_client.headers.update( + {"Authorization": f"Bearer {signed_jwt_with_unauthorized_role}"} + ) + response = authenticated_client.request(method=method, url=route.path) + assert response.status_code == 401 + assert response.json() == {"detail": "Could not validate token."} + + +@pytest.mark.parametrize( + "route", assets.management_router.routes, ids=lambda r: r.name +) +def test_invalid_jwt_signature( + route, authenticated_client, another_rsa_public_key_pem +): + """ + Endpoints should not be accessible, if the public key retrieved from + keycloak does not match the private key used to sign a JWT. + """ + assert len(route.methods) == 1, "Test assumes one method per route." + method = next(iter(route.methods)) + # The token signature of the authenticated client will not match the public + # key anymore + authenticated_client.app.dependency_overrides[ + Keycloak.get_public_key_for_workshop_realm + ] = lambda: another_rsa_public_key_pem.decode() + response = authenticated_client.request(method=method, url=route.path) + assert response.status_code == 401 + assert response.json() == {"detail": "Could not validate token."} + + +@pytest.fixture +def expired_jwt_payload(): + return { + "iat": (datetime.now(UTC) - timedelta(60)).timestamp(), + "exp": (datetime.now(UTC) - timedelta(1)).timestamp(), + "preferred_username": "user", + "realm_access": {"roles": ["assets"]} + } + + +@pytest.fixture +def expired_jwt(expired_jwt_payload, rsa_private_key_pem: bytes): + """Create an expired JWT signed with private RSA key.""" + return jws.sign( + expired_jwt_payload, rsa_private_key_pem, algorithm="RS256" + ) + + +@pytest.mark.parametrize( + "route", assets.management_router.routes, ids=lambda r: r.name +) +def test_expired_jwt(route, authenticated_client, expired_jwt): + """ + Endpoints should not be accessible, if the bearer token is expired. + """ + assert len(route.methods) == 1, "Test assumes one method per route." + method = next(iter(route.methods)) + # The token offered by the authenticated client is expired + response = authenticated_client.request( + method=method, + url=route.path, + headers={"Authorization": f"Bearer {expired_jwt}"} + ) + assert response.status_code == 401 + assert response.json() == {"detail": "Could not validate token."} + + +@pytest.fixture +def public_async_client(app, base_url): + """ + Client to access the public dataspace router. Does not have a Bearer + token issued by keycloak. + """ + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url=base_url + ) + return client + + +@pytest.mark.asyncio +async def test_get_published_dataset( + authenticated_async_client, + public_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_avoid_external_request, + tmp_path +): + asset_id = asset_ids_in_data_context[0] + async with (initialized_beanie_context, data_context): + # Get one of the assets in the data_context, process it's definition + # and have the client authenticated for management publish it + asset = await Asset.get(asset_id) + await asset.process_definition() + await authenticated_async_client.post( + f"/dataspace/manage/assets/{asset_id}/publication", + json={"nautilus_private_key": "42"} + ) + # As part of the publishing process, an asset_url and asset_key were + # created. Fetch those from the db, as the public client will need + # them to access the asset data archive. + await asset.sync() + asset_url = asset.publication.asset_url + asset_key = asset.publication.asset_key + # Confirm expectation about the constructed URL + assert asset_url == (f"{str(public_async_client.base_url)}/dataspace/" + f"public/assets/{asset_id}/data") + + # Now do the actual testing: Can the public client access the asset + # data archive using the automatically created url and key? + response = await public_async_client.get( + asset_url, + headers={"data_key": asset_key, "x-api-key": "assets-key-dev"} + ) + assert response.status_code == 200 + # Download the archive and validate structure + download_path = tmp_path / "download.zip" + with open(download_path, "wb") as file: + file.write(response.content) + with ZipFile(download_path, "r") as archive: + # Get a list of all members (files and directories) in the archive + archive_members = archive.namelist() + # Ensure the presence of the presence of the expected "cases/" and + # "signals/" directories + assert "cases/" in archive_members + assert "signals/" in archive_members + + +@pytest.mark.asyncio +async def test_get_published_dataset_invalid_asset_key( + authenticated_async_client, + public_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context, + patch_nautilus_to_avoid_external_request +): + asset_id = asset_ids_in_data_context[0] + async with (initialized_beanie_context, data_context): + # Get one of the assets in the data_context, process it's definition + # and have the client authenticated for management publish it + asset = await Asset.get(asset_id) + await asset.process_definition() + await authenticated_async_client.post( + f"/dataspace/manage/assets/{asset_id}/publication", + json={"nautilus_private_key": "42"} + ) + # As part of the publishing process, an asset_url and asset_key were + # created. Only fetch the url here. + await asset.sync() + asset_url = asset.publication.asset_url + # Confirm expectation about the constructed URL + assert asset_url == (f"{str(public_async_client.base_url)}/dataspace/" + f"public/assets/{asset_id}/data") + + # Now do the actual testing: The asset url is valid, but the public + # client can not access the dataset with an invalid asset_key + response = await public_async_client.get( + asset_url, + headers={ + "data_key": "this-sure-is-not-the-right-key", + "x-api-key": "assets-key-dev"} + ) + assert response.status_code == 401 + assert response.json() == {"detail": "Could not validate asset key."} + + +@pytest.mark.asyncio +async def test_get_published_dataset_asset_not_published( + public_async_client, + initialized_beanie_context, + data_context, + asset_ids_in_data_context +): + # Id of an unpublished asset in the data context + asset_id = asset_ids_in_data_context[0] + async with (initialized_beanie_context, data_context): + # If the asset would be published, this would be the url for data + # retrieval by the public client. + asset_url = f"/dataspace/public/assets/{asset_id}/data" + # Public client tries to fetch the data (maybe the asset was published + # in the past) + response = await public_async_client.get( + asset_url, + headers={"data_key": "some-key", "x-api-key": "assets-key-dev"} + ) + assert response.status_code == 404 + assert response.json() == { + "detail": f"No published asset with ID '{asset_id}' found." + } + + +@pytest.mark.asyncio +async def test_get_published_dataset_asset_invalid_asset_id( + public_async_client, + initialized_beanie_context +): + async with (initialized_beanie_context): + # Public client attempts to fetch an asset that does not exist at all + # (Note: No data context here and new asset id) + asset_id = str(ObjectId()) + asset_url = f"/dataspace/public/assets/{asset_id}/data" + response = await public_async_client.get( + asset_url, + headers={"data_key": "some-key", "x-api-key": "assets-key-dev"} + ) + assert response.status_code == 404 + assert response.json() == { + "detail": f"No asset with id '{asset_id}' found." + } + + +def test_get_published_dataset_no_asset_key(unauthenticated_client): + any_asset_id = str(ObjectId()) + response = unauthenticated_client.get( + f"/dataspace/public/assets/{any_asset_id}/data", + headers={"x-api-key": "assets-key-dev"} + ) + assert response.status_code == 403 + assert response.json() == {"detail": "Not authenticated"} + + +def test_get_published_dataset_no_asset_api_key(unauthenticated_client): + any_asset_id = str(ObjectId()) + response = unauthenticated_client.get( + f"/dataspace/public/assets/{any_asset_id}/data", + headers={"x-api-key": "assets-key-dev"} + ) + assert response.status_code == 403 + assert response.json() == {"detail": "Not authenticated"} diff --git a/api/tests/routers/test_customers.py b/api/tests/routers/test_customers.py index b4f9930a..63637503 100644 --- a/api/tests/routers/test_customers.py +++ b/api/tests/routers/test_customers.py @@ -32,7 +32,7 @@ def signed_jwt(jwt_payload, rsa_private_key_pem: bytes): @pytest.fixture def app(motor_db): app = FastAPI() - app.include_router(customers.router) + app.include_router(customers.router, prefix="/customers") yield app @@ -151,7 +151,7 @@ async def test_list_customers_in_empty_db( authenticated_async_client, initialized_beanie_context ): async with initialized_beanie_context: - response = await authenticated_async_client.get("/") + response = await authenticated_async_client.get("/customers") assert response.status_code == 200 assert response.json() == [] assert response.headers["link"] == "", \ @@ -167,7 +167,7 @@ async def test_list_customers( ): async with initialized_beanie_context, data_context: # Request without any additional params - response = await authenticated_async_client.get("/") + response = await authenticated_async_client.get("/customers") # Validate response status code and data assert response.status_code == 200 assert len(response.json()) == n_customers_in_data_context @@ -210,7 +210,7 @@ async def test_list_customers_pagination( await Customer(first_name="B", last_name="A").create() await Customer(first_name="A", last_name="A").create() response = await authenticated_async_client.get( - f"/?page_size={page_size}&page={page}" + f"/customers?page_size={page_size}&page={page}" ) assert response.status_code == 200 response_data = response.json() @@ -230,7 +230,7 @@ async def test_list_customers_pagination_valid_page_size_limits( ): async with initialized_beanie_context, data_context: response = await authenticated_async_client.get( - f"/?page_size={page_size}&page=0" + f"/customers?page_size={page_size}&page=0" ) assert response.status_code == 200 assert len(response.json()) == min(page_size, n_customers_in_data_context) @@ -245,7 +245,7 @@ async def test_list_customers_pagination_invalid_page_size_limits( ): async with initialized_beanie_context: response = await authenticated_async_client.get( - f"/?page_size={page_size}&page=0" + f"/customers?page_size={page_size}&page=0" ) assert response.status_code == 422, \ "Expected response to indicate unprocessable content." @@ -263,7 +263,7 @@ async def test_list_customers_out_of_range_page( out_of_range_page = max_page_index + 1 async with initialized_beanie_context, data_context: response = await authenticated_async_client.get( - f"/?page_size={page_size}&page={out_of_range_page}" + f"/customers?page_size={page_size}&page={out_of_range_page}" ) assert response.status_code == 400 assert response.json()["detail"] == \ @@ -286,7 +286,7 @@ async def test_list_customers_pagination_links( c_1 = await Customer(first_name="A", last_name="A").create() # Test retrieval of all docs using the link header for navigation retrieved_docs = [] - next_page = f"/?page_size={page_size}&page=0" + next_page = f"/customers?page_size={page_size}&page=0" while next_page: response = await authenticated_async_client.get(next_page) assert response.status_code == 200 @@ -310,7 +310,7 @@ async def test_add_customer( last_name = "some-last-name" async with initialized_beanie_context: response = await authenticated_async_client.post( - "/", + "/customers", json={"first_name": first_name, "last_name": last_name} ) assert response.status_code == 201 @@ -334,7 +334,9 @@ async def test_get_customer( ): customer_id = customer_ids_in_data_context[0] async with initialized_beanie_context, data_context: - response = await authenticated_async_client.get(f"/{customer_id}") + response = await authenticated_async_client.get( + f"/customers/{customer_id}" + ) assert response.status_code == 200 assert response.json()["_id"] == customer_id @@ -350,7 +352,7 @@ async def test_update_customer( update = {"first_name": "NewFirstName", "city": "NewCity"} async with initialized_beanie_context, data_context: response = await authenticated_async_client.patch( - f"/{customer_id}", json=update + f"/customers/{customer_id}", json=update ) assert response.status_code == 200 # Confirm customer data in response @@ -373,7 +375,9 @@ async def test_delete_customer( ): customer_id = customer_ids_in_data_context[0] async with initialized_beanie_context, data_context: - response = await authenticated_async_client.delete(f"/{customer_id}") + response = await authenticated_async_client.delete( + f"/customers/{customer_id}" + ) assert response.status_code == 200 assert response.json() is None # Confirm deletion in db @@ -392,7 +396,7 @@ async def test_customer_not_found( customer_id = str(ObjectId()) async with initialized_beanie_context: response = await authenticated_async_client.request( - method=method, url=f"/{customer_id}" + method=method, url=f"/customers/{customer_id}" ) assert response.status_code == 404 assert response.json()["detail"] == \ @@ -406,7 +410,9 @@ def test_missing_bearer_token(route, unauthenticated_client): """Endpoints should not be accessible without a bearer token.""" assert len(route.methods) == 1, "Test assumes one method per route." method = next(iter(route.methods)) - response = unauthenticated_client.request(method=method, url=route.path) + response = unauthenticated_client.request( + method=method, url=f"/customers{route.path}" + ) assert response.status_code == 403 assert response.json() == {"detail": "Not authenticated"} @@ -442,7 +448,9 @@ def test_unauthorized_user( authenticated_client.headers.update( {"Authorization": f"Bearer {signed_jwt_with_unauthorized_role}"} ) - response = authenticated_client.request(method=method, url=route.path) + response = authenticated_client.request( + method=method, url=f"/customers{route.path}" + ) assert response.status_code == 401 assert response.json() == {"detail": "Could not validate token."} @@ -464,7 +472,9 @@ def test_invalid_jwt_signature( authenticated_client.app.dependency_overrides[ Keycloak.get_public_key_for_workshop_realm ] = lambda: another_rsa_public_key_pem.decode() - response = authenticated_client.request(method=method, url=route.path) + response = authenticated_client.request( + method=method, url=f"/customers{route.path}" + ) assert response.status_code == 401 assert response.json() == {"detail": "Could not validate token."} @@ -499,7 +509,7 @@ def test_expired_jwt(route, authenticated_client, expired_jwt): # The token offered by the authenticated client is expired response = authenticated_client.request( method=method, - url=route.path, + url=f"/customers{route.path}", headers={"Authorization": f"Bearer {expired_jwt}"} ) assert response.status_code == 401 diff --git a/api/tests/routers/test_shared.py b/api/tests/routers/test_shared.py index 27554aaf..932150f7 100644 --- a/api/tests/routers/test_shared.py +++ b/api/tests/routers/test_shared.py @@ -124,9 +124,14 @@ def case_data(case_id, customer_id, vin, workshop_id): @pytest.fixture -def timeseries_data(): +def timeseries_data_component(): + return "battery" + + +@pytest.fixture +def timeseries_data(timeseries_data_component): return { - "component": "battery", + "component": timeseries_data_component, "label": "norm", "sampling_rate": 1, "duration": 3, @@ -136,9 +141,14 @@ def timeseries_data(): @pytest.fixture -def obd_data(): +def obd_data_dtc(): + return "X4242" + + +@pytest.fixture +def obd_data(obd_data_dtc): return { - "dtcs": ["P0001", "U0001"] + "dtcs": ["P0001", "U0001", obd_data_dtc] } @@ -235,7 +245,13 @@ async def test_list_cases( assert response_data[0]["_id"] == case_id -@pytest.mark.parametrize("query_param", ["customer_id", "vin", "workshop_id"]) +@pytest.mark.parametrize( + "query_param", + [ + "customer_id", "vin", "workshop_id", "obd_data_dtc", + "timeseries_data_component" + ] +) @pytest.mark.asyncio async def test_list_cases_with_single_filter( authenticated_async_client, initialized_beanie_context, data_context, @@ -256,12 +272,15 @@ async def test_list_cases_with_single_filter( @pytest.mark.asyncio async def test_list_cases_with_multiple_filters( authenticated_async_client, initialized_beanie_context, data_context, - case_id, customer_id, vin, workshop_id + case_id, customer_id, vin, workshop_id, obd_data_dtc, + timeseries_data_component ): """Test filtering by multiple query params.""" query_string = f"?customer_id={customer_id}&" \ f"vin={vin}&" \ - f"workshop_id={workshop_id}" + f"workshop_id={workshop_id}&" \ + f"obd_data_dtc={obd_data_dtc}&" \ + f"timeseries_data_component={timeseries_data_component}" url = f"/cases{query_string}" async with initialized_beanie_context, data_context: response = await authenticated_async_client.get(url) @@ -271,7 +290,11 @@ async def test_list_cases_with_multiple_filters( assert response_data[0]["_id"] == case_id -@pytest.mark.parametrize("query_param", ["customer_id", "vin", "workshop_id"]) +@pytest.mark.parametrize( + "query_param", + ["customer_id", "vin", "workshop_id", "obd_data_dtc", + "timeseries_data_component"] +) @pytest.mark.asyncio async def test_list_cases_with_unmatched_filters( authenticated_async_client, initialized_beanie_context, data_context, diff --git a/api/tests/routers/test_workshop.py b/api/tests/routers/test_workshop.py index 25470d6d..3e51de73 100644 --- a/api/tests/routers/test_workshop.py +++ b/api/tests/routers/test_workshop.py @@ -154,8 +154,13 @@ def authenticated_client( @mock.patch("api.routers.workshop.Case.find_in_hub", autospec=True) def test_list_cases(find_in_hub, authenticated_client, workshop_id): - - async def mock_find_in_hub(customer_id, workshop_id, vin): + async def mock_find_in_hub( + customer_id, + workshop_id, + vin, + obd_data_dtc, + timeseries_data_component + ): return [] # patch Case.find_in_hub to use mock_find_in_hub @@ -168,7 +173,11 @@ async def mock_find_in_hub(customer_id, workshop_id, vin): assert response.status_code == 200 assert response.json() == [] find_in_hub.assert_called_once_with( - customer_id=None, vin=None, workshop_id=workshop_id + customer_id=None, + vin=None, + workshop_id=workshop_id, + obd_data_dtc=None, + timeseries_data_component=None ) @@ -176,8 +185,13 @@ async def mock_find_in_hub(customer_id, workshop_id, vin): def test_list_cases_with_filters( find_in_hub, authenticated_client, workshop_id ): - - async def mock_find_in_hub(customer_id, workshop_id, vin): + async def mock_find_in_hub( + customer_id, + workshop_id, + vin, + obd_data_dtc, + timeseries_data_component + ): return [] # patch Case.find_in_hub to use mock_find_in_hub @@ -186,16 +200,27 @@ async def mock_find_in_hub(customer_id, workshop_id, vin): # request with filter params customer_id = "test customer" vin = "test vin" + obd_data_dtc = "P0001" + timeseries_data_component = "Test-Comp" response = authenticated_client.get( f"/{workshop_id}/cases", - params={"customer_id": customer_id, "vin": vin} + params={ + "customer_id": customer_id, + "vin": vin, + "obd_data_dtc": obd_data_dtc, + "timeseries_data_component": timeseries_data_component + } ) # confirm expected response and usage of db interface assert response.status_code == 200 assert response.json() == [] find_in_hub.assert_called_once_with( - customer_id=customer_id, vin=vin, workshop_id=workshop_id + customer_id=customer_id, + vin=vin, + workshop_id=workshop_id, + obd_data_dtc=obd_data_dtc, + timeseries_data_component=timeseries_data_component ) @@ -233,7 +258,6 @@ def test_add_case_with_invalid_customer_id( @mock.patch("api.routers.workshop.Case.get", autospec=True) @pytest.mark.asyncio async def test_case_from_workshop(get, case_id, workshop_id): - async def mock_get(*args): """ Always returns a case with case_id and workshop_id predefined in test @@ -270,7 +294,6 @@ async def mock_get(*args): @mock.patch("api.routers.workshop.Case.get", autospec=True) @pytest.mark.asyncio async def test_case_from_workshop_wrong_workshop(get, case_id, workshop_id): - async def mock_get(*args): """ Always returns a case with case_id as predifined in test scope above diff --git a/demo-ui/demo_ui/main.py b/demo-ui/demo_ui/main.py index 2ef7d99b..8c6b412f 100644 --- a/demo-ui/demo_ui/main.py +++ b/demo-ui/demo_ui/main.py @@ -22,7 +22,7 @@ def filter(self, record: logging.LogRecord) -> bool: return record.getMessage().find(self.prefix) == -1 -app = FastAPI() +app = FastAPI(root_path=settings.root_path) app.add_middleware( SessionMiddleware, secret_key=settings.session_secret, max_age=None ) @@ -232,7 +232,9 @@ def login_post( ) request.session["access_token"] = access_token request.session["refresh_token"] = refresh_token - redirect_url = app.url_path_for("cases", workshop_id=workshop_id) + redirect_url = app.root_path + app.url_path_for( + "cases", workshop_id=workshop_id + ) return redirect_url @@ -287,7 +289,7 @@ async def new_case_post( # remove empty fields form = {k: v for k, v in form.items() if v} case = await post_to_api(ressource_url, access_token, json=dict(form)) - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "case", workshop_id=case["workshop_id"], case_id=case["_id"] ) return redirect_url @@ -330,7 +332,7 @@ def case_delete_get( ressource_url: str = Depends(get_case_url) ): delete_via_api(ressource_url, access_token) - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "cases", workshop_id=request.path_params["workshop_id"] ) return redirect_url @@ -373,7 +375,7 @@ async def new_obd_data_post( ) new_data_id = case["obd_data"][-1]["data_id"] - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "obd_data", workshop_id=case["workshop_id"], case_id=case["_id"], @@ -413,7 +415,7 @@ def obd_data_delete_get( ressource_url: str = Depends(get_obd_data_url) ): delete_via_api(ressource_url, access_token) - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "case", workshop_id=request.path_params["workshop_id"], case_id=request.path_params["case_id"] @@ -465,7 +467,7 @@ async def new_timeseries_data_upload_omniview( ) new_data_id = case["timeseries_data"][-1]["data_id"] - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "timeseries_data", workshop_id=case["workshop_id"], case_id=case["_id"], @@ -497,7 +499,7 @@ async def new_timeseries_data_upload_picoscope( ) new_data_id = case["timeseries_data"][-1]["data_id"] - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "timeseries_data", workshop_id=case["workshop_id"], case_id=case["_id"], @@ -543,7 +545,7 @@ def timeseries_data_delete_get( ressource_url: str = Depends(get_timeseries_data_url) ): delete_via_api(ressource_url, access_token) - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "case", workshop_id=request.path_params["workshop_id"], case_id=request.path_params["case_id"] @@ -582,7 +584,7 @@ async def new_symptom_post( ): form = await request.form() case = await post_to_api(ressource_url, access_token, json=dict(form)) - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "case", workshop_id=case["workshop_id"], case_id=case["_id"] ) return redirect_url @@ -599,7 +601,7 @@ def symptom_delete_get( ressource_url: str = Depends(get_symptoms_url) ): delete_via_api(ressource_url, access_token) - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "case", workshop_id=request.path_params["workshop_id"], case_id=request.path_params["case_id"] @@ -618,7 +620,7 @@ async def start_diagnosis( ressource_url: str = Depends(get_diagnosis_url) ): await post_to_api(ressource_url, access_token=access_token) - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "case", workshop_id=request.path_params["workshop_id"], case_id=request.path_params["case_id"] @@ -678,7 +680,7 @@ def diagnosis_delete_get( ressource_url: str = Depends(get_diagnosis_url) ): delete_via_api(ressource_url, access_token) - redirect_url = app.url_path_for( + redirect_url = app.root_path + app.url_path_for( "case", workshop_id=request.path_params["workshop_id"], case_id=request.path_params["case_id"] @@ -686,9 +688,10 @@ def diagnosis_delete_get( return redirect_url -@app.get("/ui/logout", response_class=RedirectResponse) +@app.get("/ui/logout", response_class=RedirectResponse, status_code=303) def logout(request: Request): request.session.pop("access_token", None) request.session.pop("refresh_token", None) flash_message(request, "Sie wurden erfolgreich ausgeloggt.") - return RedirectResponse("/ui", status_code=303) + redirect_url = app.root_path + app.url_path_for("login_get") + return redirect_url diff --git a/demo-ui/demo_ui/settings.py b/demo-ui/demo_ui/settings.py index 965f4ef0..462676f8 100644 --- a/demo-ui/demo_ui/settings.py +++ b/demo-ui/demo_ui/settings.py @@ -16,5 +16,7 @@ class Settings(BaseSettings): timezone: str = "Europe/Berlin" + root_path: str = "" + settings = Settings() diff --git a/demo-ui/demo_ui/templates/case.html b/demo-ui/demo_ui/templates/case.html index 7dfab55e..737a00c7 100644 --- a/demo-ui/demo_ui/templates/case.html +++ b/demo-ui/demo_ui/templates/case.html @@ -1,13 +1,13 @@ {% extends "base.html" %} {% block sidebar %} -Werkstatt {{ request.path_params["workshop_id"] }} (Abmelden) -Fälle +Werkstatt {{ request.path_params["workshop_id"] }} (Abmelden) +Fälle Ausgewählt: {{ case["_id"] }} {% endblock %} {% block content %} -ID | diff --git a/demo-ui/demo_ui/templates/diagnosis_report.html b/demo-ui/demo_ui/templates/diagnosis_report.html index d1dc62b6..deb1083a 100644 --- a/demo-ui/demo_ui/templates/diagnosis_report.html +++ b/demo-ui/demo_ui/templates/diagnosis_report.html @@ -1,16 +1,16 @@ {% extends "base.html" %} {% block sidebar %} -Werkstatt {{ request.path_params["workshop_id"] }} (Abmelden) -Fälle -Ausgewählt: {{ case_id }} +Werkstatt {{ request.path_params["workshop_id"] }} (Abmelden) +Fälle +Ausgewählt: {{ case_id }} Diagnose {% endblock %} {% block content %}
---|