Merge branch 'main' into feature/mula/task-events

* main: (25 commits)
  Create object history API (#2074)
  Bump actions/github-script from 6 to 7 (#2076)
  Installation manual for Windows (2) (#2096)
  Update howdoesitwork.rst (#2091)
  Add benchmarking script to the scheduler (#2071)
  Add fix-poetry-merge-conflict makefile command (#2088)
  Bump sphinx-rtd-theme from 1.2.2 to 2.0.0 (#2080)
  Lower quality level so the CI check doesn't fail (#2086)
  Update xtdb version in octopoes CI docker compose and docker-compose.release-example.yml (#2085)
  Name test nodes by testname instead of uuid (#2087)
  Upgrade to Pydantic v2 (#1912)
  Docs: add dependency installation commands for RHEL based systems (#2059)
  Fix/2072 (#2082)
  Feature/service to systems reports rocky (#2073)
  Update scheduler python packages (#2062)
  Add uvicorn back as non-dev dependency (#2053)
  Bump `cryptography` (#2070)
  Filter tree objects with depth=1 for Findings  (#1982)
  Bump aiohttp from 3.8.6 to 3.9.0 in /boefjes (#2061)
  Translations update from Hosted Weblate (#2057)
  ...

.github/workflows/build-rdo-package.yml

@@ -15,7 +15,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Generic bits
     steps:
@@ -26,7 +26,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.10'
           cache: 'pip'
 
       - name: Prep Install requirements
@@ -48,12 +48,12 @@ jobs:
 
       - name: Octopoes Build whl package
         run: |
-          python3.8 -m pip install build
-          python3.8 -m build
+          python3.10 -m pip install build
+          python3.10 -m build
         working-directory: ./octopoes
 
       - name: Octopoes Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Octopoes Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
@@ -84,7 +84,7 @@ jobs:
         working-directory: ./rocky
 
       - name: Rocky Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Rocky Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
@@ -113,7 +113,7 @@ jobs:
         working-directory: ./rocky
 
       - name: Rocky Compilemessages
-        run: /var/www/html/.venv/bin/python3.8 manage.py collectstatic && /var/www/html/.venv/bin/python3.8 manage.py compress && /var/www/html/.venv/bin/python3.8 manage.py compilemessages
+        run: /var/www/html/.venv/bin/python3.10 manage.py collectstatic && /var/www/html/.venv/bin/python3.10 manage.py compress && /var/www/html/.venv/bin/python3.10 manage.py compilemessages
         working-directory: ./rocky
         env:
           BYTES_API: http://bytes:8000
@@ -138,7 +138,7 @@ jobs:
         working-directory: ./bytes
 
       - name: Bytes Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Bytes Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
@@ -159,14 +159,14 @@ jobs:
         working-directory: ./mula
 
       - name: Mula Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Create scheduler release archive
         run: tar -cvzf ${{ env.PKGDIR }}/scheduler_${{ env.RELEASE_VERSION }}.tar.gz --exclude=./.git* --exclude=Makefile --exclude=Dockerfile --exclude=base.yml --exclude=requirements* --exclude=tests .
         working-directory: ./mula
 
       - name: Create virtual env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
@@ -184,7 +184,7 @@ jobs:
         working-directory: ./boefjes
 
       - name: Boefjes Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; find . -name requirements.txt | xargs -L 1 pip install -r; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
@@ -206,7 +206,7 @@ jobs:
         working-directory: ./keiko
 
       - name: Keiko Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Keiko Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; find . -name requirements.txt | xargs -L 1 pip install -r

.github/workflows/octopoes_coverage.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Download artifact'
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           script: |
             let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({

.github/workflows/sigrid-pullrequest.yml

@@ -12,6 +12,7 @@ jobs:
         with:
           customer: vws
           system: kat
+          targetquality: 2.5
         env:
           SIGRID_CI_TOKEN: "${{ secrets.SIGRID_CI_TOKEN }}"
 #      - name: "Sigrid pull request feedback"

Makefile

@@ -122,3 +122,14 @@ poetry-dependencies:
 		poetry export -C $$path --without=dev -f requirements.txt -o $$path/requirements.txt; \
 		poetry export -C $$path --with=dev -f requirements.txt -o $$path/requirements-dev.txt; \
 	done
+
+fix-poetry-merge-conflict:
+	for path in `git diff --staged --name-only | grep pyproject | cut -d / -f 1`;do \
+		echo $$path; \
+		git restore --staged $$path/poetry.lock $$path/requirements*; \
+		git checkout --theirs $$path/poetry.lock $$path/requirements*; \
+		poetry lock --no-update -C $$path; \
+		poetry export -C $$path --without=dev -f requirements.txt -o $$path/requirements.txt; \
+		poetry export -C $$path --with=dev -f requirements.txt -o $$path/requirements-dev.txt; \
+		git add $$path/poetry.lock $$path/requirements*; \
+	done

boefjes/boefjes/api.py

@@ -6,7 +6,7 @@
 from typing import List, Optional
 
 from fastapi import Depends, FastAPI, HTTPException, Response
-from pydantic import BaseModel, Extra, Field
+from pydantic import BaseModel, ConfigDict, Field
 from requests import HTTPError
 from uvicorn import Config, Server
 
@@ -52,9 +52,7 @@ class BoefjeInput(BaseModel):
     task_id: str
     output_url: str
     boefje_meta: BoefjeMeta
-
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class StatusEnum(str, Enum):
@@ -63,18 +61,18 @@ class StatusEnum(str, Enum):
 
 
 class File(BaseModel):
-    name: Optional[str]
+    name: Optional[str] = None
     content: str = Field(..., contentEncoding="base64")
-    tags: Optional[List[str]]
+    tags: Optional[List[str]] = None
 
 
 class BoefjeOutput(BaseModel):
     status: StatusEnum
-    files: Optional[List[File]]
+    files: Optional[List[File]] = None
 
 
 def get_scheduler_client():
-    return SchedulerAPIClient(settings.scheduler_api)
+    return SchedulerAPIClient(str(settings.scheduler_api))
 
 
 def get_bytes_client():
@@ -103,7 +101,7 @@ async def boefje_input(
 
     boefje_meta = create_boefje_meta(task, local_repository)
 
-    output_url = settings.boefje_api + "/api/v0/tasks/" + task_id
+    output_url = str(settings.boefje_api).rstrip("/") + f"/api/v0/tasks/{task_id}"
     return BoefjeInput(task_id=task_id, output_url=output_url, boefje_meta=boefje_meta)
 
 

boefjes/boefjes/app.py

@@ -260,7 +260,7 @@ def get_runtime_manager(settings: Settings, queue: WorkerManager.Queue, log_leve
 
     return SchedulerWorkerManager(
         item_handler,
-        SchedulerAPIClient(settings.scheduler_api),  # Do not share a session between workers
+        SchedulerAPIClient(str(settings.scheduler_api)),  # Do not share a session between workers
         settings,
         log_level,
     )

boefjes/boefjes/clients/bytes_client.py

@@ -19,7 +19,7 @@ class BytesAPISession(requests.Session):
     def __init__(self, base_url: str):
         super().__init__()
 
-        self._base_url = base_url
+        self._base_url = base_url.rstrip("/")
         self.headers["User-Agent"] = f"bytes-api-client/{BYTES_API_CLIENT_VERSION}"
 
     def request(self, method: str, url: Union[str, bytes], **kwargs) -> requests.Response:  # type: ignore
@@ -94,8 +94,7 @@ def get_boefje_meta(self, boefje_meta_id: str) -> BoefjeMeta:
         response = self._session.get(f"/bytes/boefje_meta/{boefje_meta_id}", headers=self.headers)
         self._verify_response(response)
 
-        boefje_meta_json = response.json()
-        return BoefjeMeta.parse_obj(boefje_meta_json)
+        return BoefjeMeta.model_validate_json(response.content)
 
     @retry_with_login
     def save_normalizer_meta(self, normalizer_meta: NormalizerMeta) -> None:
@@ -131,4 +130,4 @@ def get_raw_meta(self, raw_data_id: str) -> RawDataMeta:
         response = self._session.get(f"/bytes/raw/{raw_data_id}/meta", headers=self.headers)
         self._verify_response(response)
 
-        return RawDataMeta.parse_obj(response.json())
+        return RawDataMeta.model_validate_json(response.content)

boefjes/boefjes/clients/scheduler_client.py

@@ -5,7 +5,7 @@
 from typing import List, Optional, Union
 
 import requests
-from pydantic import BaseModel, parse_obj_as
+from pydantic import BaseModel, TypeAdapter
 from requests.adapters import HTTPAdapter, Retry
 
 from boefjes.job_models import BoefjeMeta, NormalizerMeta
@@ -26,7 +26,7 @@ class QueuePrioritizedItem(BaseModel):
 
     id: uuid.UUID
     priority: int
-    hash: Optional[str]
+    hash: Optional[str] = None
     data: Union[BoefjeMeta, NormalizerMeta]
 
 
@@ -80,7 +80,7 @@ def __init__(self, *args, skip_log=False, **kwargs):
 
 class SchedulerAPIClient(SchedulerClientInterface):
     def __init__(self, base_url: str):
-        self.base_url = base_url
+        self.base_url = base_url.rstrip("/")
         self._session = requests.Session()
 
         max_retries = LogRetry(skip_log=True, total=6, backoff_factor=1)
@@ -95,13 +95,13 @@ def get_queues(self) -> List[Queue]:
         response = self._session.get(f"{self.base_url}/queues")
         self._verify_response(response)
 
-        return parse_obj_as(List[Queue], response.json())
+        return TypeAdapter(List[Queue]).validate_json(response.content)
 
     def pop_item(self, queue: str) -> Optional[QueuePrioritizedItem]:
         response = self._session.post(f"{self.base_url}/queues/{queue}/pop")
         self._verify_response(response)
 
-        return parse_obj_as(Optional[QueuePrioritizedItem], response.json())
+        return TypeAdapter(Optional[QueuePrioritizedItem]).validate_json(response.content)
 
     def push_item(self, queue_id: str, p_item: QueuePrioritizedItem) -> None:
         response = self._session.post(f"{self.base_url}/queues/{queue_id}/push", data=p_item.json())
@@ -115,4 +115,4 @@ def get_task(self, task_id: str) -> Task:
         response = self._session.get(f"{self.base_url}/tasks/{task_id}")
         self._verify_response(response)
 
-        return parse_obj_as(Task, response.json())
+        return Task.model_validate_json(response.content)

boefjes/boefjes/config.py

@@ -1,10 +1,11 @@
 import logging
 import os
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, Optional, Tuple, Type
 
-from pydantic import AmqpDsn, AnyHttpUrl, BaseSettings, Field, FilePath, IPvAnyAddress, PostgresDsn
-from pydantic.env_settings import SettingsSourceCallable
+from pydantic import AmqpDsn, AnyHttpUrl, Field, FilePath, IPvAnyAddress, PostgresDsn
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
+from pydantic_settings.sources import EnvSettingsSource
 
 BASE_DIR: Path = Path(__file__).parent.resolve()
 
@@ -13,15 +14,15 @@
     BASE_DIR = Path("../")
 
 
-class BackwardsCompatibleEnvSettings:
+class BackwardsCompatibleEnvSettings(EnvSettingsSource):
     backwards_compatibility_mapping = {
         "LOG_CFG": "BOEFJES_LOG_CFG",
     }
 
-    def __call__(self, settings: BaseSettings) -> Dict[str, Any]:
+    def __call__(self) -> Dict[str, Any]:
         d: Dict[str, Any] = {}
         env_vars = {k.lower(): v for k, v in os.environ.items()}
-        env_prefix = settings.__config__.env_prefix.lower()
+        env_prefix = self.settings_cls.model_config.get("env_prefix", "").lower()
 
         for old_name, new_name in self.backwards_compatibility_mapping.items():
             old_name, new_name = old_name.lower(), new_name.lower()
@@ -48,25 +49,27 @@ class Settings(BaseSettings):
     )
 
     # Queue configuration
-    queue_uri: AmqpDsn = Field(..., description="KAT queue URI", example="amqp://", env="QUEUE_URI")
+    queue_uri: AmqpDsn = Field(..., description="KAT queue URI", examples=["amqp://"], validation_alias="QUEUE_URI")
 
     katalogus_db_uri: PostgresDsn = Field(
         ...,
-        example="postgresql://xx:xx@host:5432/katalogus",
+        examples=["postgresql://xx:xx@host:5432/katalogus"],
         description="Katalogus Postgres DB URI",
-        env="KATALOGUS_DB_URI",
+        validation_alias="KATALOGUS_DB_URI",
     )
 
     scheduler_api: AnyHttpUrl = Field(
-        ..., example="http://localhost:8004", description="Mula API URL", env="SCHEDULER_API"
+        ..., examples=["http://localhost:8004"], description="Mula API URL", validation_alias="SCHEDULER_API"
     )
     katalogus_api: AnyHttpUrl = Field(
-        ..., example="http://localhost:8003", description="Katalogus API URL", env="KATALOGUS_API"
+        ..., examples=["http://localhost:8003"], description="Katalogus API URL", validation_alias="KATALOGUS_API"
     )
     octopoes_api: AnyHttpUrl = Field(
-        ..., example="http://localhost:8001", description="Octopoes API URL", env="OCTOPOES_API"
+        ..., examples=["http://localhost:8001"], description="Octopoes API URL", validation_alias="OCTOPOES_API"
+    )
+    boefje_api: AnyHttpUrl = Field(
+        ..., examples=["http://boefje:8000"], description="Boefje API URL", validation_alias="BOEFJE_API"
     )
-    boefje_api: AnyHttpUrl = Field(..., example="http://boefje:8000", description="Boefje API URL", env="BOEFJE_API")
     # Boefje server settings
     boefje_api_host: str = Field(
         "0.0.0.0",
@@ -76,33 +79,38 @@ class Settings(BaseSettings):
         8000,
         description="Host port of the Boefje API server",
     )
-
     boefje_docker_network: str = Field(
         "bridge",
         description="Docker network to run Boefjes in",
         env="BOEFJE_DOCKER_NETWORK",
     )
-
-    bytes_api: AnyHttpUrl = Field(..., example="http://localhost:8002", description="Bytes API URL", env="BYTES_API")
-    bytes_username: str = Field(..., example="test", description="Bytes JWT login username", env="BYTES_USERNAME")
-    bytes_password: str = Field(..., example="secret", description="Bytes JWT login password", env="BYTES_PASSWORD")
+    bytes_api: AnyHttpUrl = Field(
+        ..., examples=["http://localhost:8002"], description="Bytes API URL", validation_alias="BYTES_API"
+    )
+    bytes_username: str = Field(
+        ..., examples=["test"], description="Bytes JWT login username", validation_alias="BYTES_USERNAME"
+    )
+    bytes_password: str = Field(
+        ..., examples=["secret"], description="Bytes JWT login password", validation_alias="BYTES_PASSWORD"
+    )
 
     span_export_grpc_endpoint: Optional[AnyHttpUrl] = Field(
-        None, description="OpenTelemetry endpoint", env="SPAN_EXPORT_GRPC_ENDPOINT"
+        None, description="OpenTelemetry endpoint", validation_alias="SPAN_EXPORT_GRPC_ENDPOINT"
     )
 
-    class Config:
-        env_prefix = "BOEFJES_"
-
-        @classmethod
-        def customise_sources(
-            cls,
-            init_settings: SettingsSourceCallable,
-            env_settings: SettingsSourceCallable,
-            file_secret_settings: SettingsSourceCallable,
-        ) -> Tuple[SettingsSourceCallable, ...]:
-            backwards_compatible_settings = BackwardsCompatibleEnvSettings()
-            return env_settings, init_settings, file_secret_settings, backwards_compatible_settings
+    model_config = SettingsConfigDict(env_prefix="BOEFJES_")
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        backwards_compatible_settings = BackwardsCompatibleEnvSettings(settings_cls)
+        return env_settings, init_settings, file_secret_settings, backwards_compatible_settings
 
 
 # Do not initialize the settings module when compiling environment docs