From d3e69248a39b5a984626937a3c4a7e7ba82a904f Mon Sep 17 00:00:00 2001 From: DecFox <33030671+DecFox@users.noreply.github.com> Date: Fri, 21 Jun 2024 11:49:54 +0530 Subject: [PATCH] feat: oonifindings service (#850) * feat(oonifindings): findings ooniapi service Create oonifindings service with clickhouse support * refactor: switch to using pytest-docker * feat: add build files * feat: oonifindings service v1 * integrate changes from common into oonirun * migrate ooniauth to new common * code cleanup * add github action for oonifindings tests * introduce api tests for oonifindings * add tests for oonifindings router * extend router tests for oonifindings * refactor: remove null account and use client * refactor: use cliet without token * delete unrequired fixtures * test for cache headers in response --- .../workflows/test_ooniapi_oonifindings.yml | 25 + ooniapi/common/src/common/auth.py | 70 +++ ooniapi/common/src/common/clickhouse_utils.py | 2 +- ooniapi/common/src/common/config.py | 1 + ooniapi/common/src/common/dependencies.py | 5 +- ooniapi/common/src/common/utils.py | 62 +-- .../ooniauth/src/ooniauth/routers/v1.py | 7 +- .../ooniauth/src/ooniauth/routers/v2.py | 7 +- .../services/ooniauth/src/ooniauth/utils.py | 2 +- .../services/ooniauth/tests/test_auth_v1.py | 2 - .../services/ooniauth/tests/test_auth_v2.py | 2 - ooniapi/services/oonifindings/.dockerignore | 10 + ooniapi/services/oonifindings/.gitignore | 3 + ooniapi/services/oonifindings/Dockerfile | 29 ++ ooniapi/services/oonifindings/LICENSE.txt | 26 + ooniapi/services/oonifindings/Makefile | 64 +++ ooniapi/services/oonifindings/README.md | 21 + ooniapi/services/oonifindings/buildspec.yml | 29 ++ .../migrations/clickhouse_init_tables.sql | 28 + ooniapi/services/oonifindings/pyproject.toml | 106 ++++ .../oonifindings/scripts/docker-smoketest.sh | 34 ++ .../src/oonifindings/__about__.py | 2 + .../oonifindings/src/oonifindings/__init__.py | 0 .../oonifindings/src/oonifindings/common | 1 + .../src/oonifindings/dependencies.py | 15 + .../oonifindings/src/oonifindings/main.py | 110 ++++ .../src/oonifindings/routers/v1.py | 479 ++++++++++++++++++ .../services/oonifindings/tests/__init__.py | 0 .../services/oonifindings/tests/conftest.py | 139 +++++ .../oonifindings/tests/docker-compose.yml | 6 + .../migrations/clickhouse_init_tables.sql | 28 + .../oonifindings/tests/test_integration.py | 42 ++ .../services/oonifindings/tests/test_main.py | 35 ++ .../oonifindings/tests/test_oonifindings.py | 425 ++++++++++++++++ .../oonirun/src/oonirun/routers/v2.py | 20 +- 35 files changed, 1772 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/test_ooniapi_oonifindings.yml create mode 100644 ooniapi/common/src/common/auth.py create mode 100644 ooniapi/services/oonifindings/.dockerignore create mode 100644 ooniapi/services/oonifindings/.gitignore create mode 100644 ooniapi/services/oonifindings/Dockerfile create mode 100644 ooniapi/services/oonifindings/LICENSE.txt create mode 100644 ooniapi/services/oonifindings/Makefile create mode 100644 ooniapi/services/oonifindings/README.md create mode 100644 ooniapi/services/oonifindings/buildspec.yml create mode 100644 ooniapi/services/oonifindings/migrations/clickhouse_init_tables.sql create mode 100644 ooniapi/services/oonifindings/pyproject.toml create mode 100644 ooniapi/services/oonifindings/scripts/docker-smoketest.sh create mode 100644 ooniapi/services/oonifindings/src/oonifindings/__about__.py create mode 100644 ooniapi/services/oonifindings/src/oonifindings/__init__.py create mode 120000 ooniapi/services/oonifindings/src/oonifindings/common create mode 100644 ooniapi/services/oonifindings/src/oonifindings/dependencies.py create mode 100644 ooniapi/services/oonifindings/src/oonifindings/main.py create mode 100644 ooniapi/services/oonifindings/src/oonifindings/routers/v1.py create mode 100644 ooniapi/services/oonifindings/tests/__init__.py create mode 100644 ooniapi/services/oonifindings/tests/conftest.py create mode 100644 ooniapi/services/oonifindings/tests/docker-compose.yml create mode 100644 ooniapi/services/oonifindings/tests/migrations/clickhouse_init_tables.sql create mode 100644 ooniapi/services/oonifindings/tests/test_integration.py create mode 100644 ooniapi/services/oonifindings/tests/test_main.py create mode 100644 ooniapi/services/oonifindings/tests/test_oonifindings.py diff --git a/.github/workflows/test_ooniapi_oonifindings.yml b/.github/workflows/test_ooniapi_oonifindings.yml new file mode 100644 index 00000000..3c605b2b --- /dev/null +++ b/.github/workflows/test_ooniapi_oonifindings.yml @@ -0,0 +1,25 @@ +name: test ooniapi/oonifindings +on: push +jobs: + run_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install hatch + run: pip install hatch + + - name: Run all tests + run: make test-cov + working-directory: ./ooniapi/services/oonifindings/ + + - name: Upload coverage to codecov + uses: codecov/codecov-action@v3 + with: + flags: oonifindings + working-directory: ./ooniapi/services/oonifindings/ diff --git a/ooniapi/common/src/common/auth.py b/ooniapi/common/src/common/auth.py new file mode 100644 index 00000000..e6740bf7 --- /dev/null +++ b/ooniapi/common/src/common/auth.py @@ -0,0 +1,70 @@ +import hashlib +from typing import Optional, Dict, Any +import jwt + + +def hash_email_address(email_address: str, key: str) -> str: + em = email_address.encode() + return hashlib.blake2b(em, key=key.encode("utf-8"), digest_size=16).hexdigest() + + +def check_email_address( + authorization: str, + jwt_encryption_key: str, + email_address: str, + key: str +) -> bool: + account_id = get_account_id_or_raise(authorization, jwt_encryption_key=jwt_encryption_key) + hashed = hash_email_address(email_address, key=key) + if account_id == hashed: + return True + return False + + +def decode_jwt(token: str, key: str, **kw) -> Dict[str, Any]: + tok = jwt.decode(token, key, algorithms=["HS256"], **kw) + return tok + + +def create_jwt(payload: dict, key: str) -> str: + token = jwt.encode(payload, key, algorithm="HS256") + if isinstance(token, bytes): + return token.decode() + else: + return token + + +def get_client_token(authorization: str, jwt_encryption_key: str): + try: + assert authorization.startswith("Bearer ") + token = authorization[7:] + return decode_jwt(token, audience="user_auth", key=jwt_encryption_key) + except: + return None + + +def get_client_role(authorization: str, jwt_encryption_key: str) -> str: + """Raise exception for unlogged users""" + tok = get_client_token(authorization, jwt_encryption_key) + try: + assert tok + return tok["role"] + except: + return None + +def get_account_id_or_none( + authorization: str, jwt_encryption_key: str +) -> Optional[str]: + """Returns None for unlogged users""" + tok = get_client_token(authorization, jwt_encryption_key) + if tok: + return tok["account_id"] + return None + + +def get_account_id_or_raise(authorization: str, jwt_encryption_key: str) -> str: + """Raise exception for unlogged users""" + tok = get_client_token(authorization, jwt_encryption_key) + if tok: + return tok["account_id"] + raise Exception diff --git a/ooniapi/common/src/common/clickhouse_utils.py b/ooniapi/common/src/common/clickhouse_utils.py index 221041b3..71407aa4 100644 --- a/ooniapi/common/src/common/clickhouse_utils.py +++ b/ooniapi/common/src/common/clickhouse_utils.py @@ -61,7 +61,7 @@ def optimize_table(db: clickhouse_driver.Client, tblname: str) -> None: def raw_query( db: clickhouse_driver.Client, query: Query, query_params: dict, query_prio=1 -): +) -> int: settings = {"priority": query_prio, "max_execution_time": 300} q = db.execute(query, query_params, with_column_types=True, settings=settings) return q diff --git a/ooniapi/common/src/common/config.py b/ooniapi/common/src/common/config.py index 431b6213..c47a6cea 100644 --- a/ooniapi/common/src/common/config.py +++ b/ooniapi/common/src/common/config.py @@ -16,6 +16,7 @@ class Settings(BaseSettings): jwt_encryption_key: str = "CHANGEME" prometheus_metrics_password: str = "CHANGEME" account_id_hashing_key: str = "CHANGEME" + collector_id: str = "CHANGEME" session_expiry_days: int = 10 login_expiry_days: int = 10 diff --git a/ooniapi/common/src/common/dependencies.py b/ooniapi/common/src/common/dependencies.py index 476a6910..22beafa4 100644 --- a/ooniapi/common/src/common/dependencies.py +++ b/ooniapi/common/src/common/dependencies.py @@ -3,7 +3,7 @@ from fastapi import Depends from fastapi import HTTPException, Header -from .utils import get_client_token +from .auth import get_client_token from .config import Settings @@ -29,6 +29,9 @@ async def verify_jwt( tok = get_client_token(authorization, settings.jwt_encryption_key) except: raise HTTPException(detail="Authentication required", status_code=401) + + if not tok: + raise HTTPException(detail="Authentication required", status_code=401) if tok["role"] not in roles: raise HTTPException(detail="Role not authorized", status_code=401) diff --git a/ooniapi/common/src/common/utils.py b/ooniapi/common/src/common/utils.py index 63bca052..c1a62fd3 100644 --- a/ooniapi/common/src/common/utils.py +++ b/ooniapi/common/src/common/utils.py @@ -1,11 +1,12 @@ from csv import DictWriter from io import StringIO +from sys import byteorder +from os import urandom import logging -from typing import Any, Dict, List, Optional, Union +from typing import List +from fastapi import Response from fastapi.responses import JSONResponse -import jwt - log = logging.getLogger(__name__) @@ -31,6 +32,15 @@ def jerror(msg, code=400, **kw) -> JSONResponse: return JSONResponse(content=dict(msg=msg, **kw), status_code=code, headers=headers) +def setcacheresponse(interval: str, response: Response): + max_age = int(interval[:-1]) * INTERVAL_UNITS[interval[-1]] + response.headers["Cache-Control"] = f"max-age={max_age}" + + +def setnocacheresponse(response: Response): + response.headers["Cache-Control"] = "no-cache" + + def commasplit(p: str) -> List[str]: assert p is not None out = set(p.split(",")) @@ -60,44 +70,10 @@ def convert_to_csv(r) -> str: return result -def decode_jwt(token: str, key: str, **kw) -> Dict[str, Any]: - tok = jwt.decode(token, key, algorithms=["HS256"], **kw) - return tok - - -def create_jwt(payload: dict, key: str) -> str: - token = jwt.encode(payload, key, algorithm="HS256") - if isinstance(token, bytes): - return token.decode() - else: - return token - - -def get_client_token(authorization: str, jwt_encryption_key: str) -> Dict[str, Any]: - assert authorization.startswith("Bearer ") - token = authorization[7:] - return decode_jwt(token, audience="user_auth", key=jwt_encryption_key) - - -def get_client_role(authorization: str, jwt_encryption_key: str) -> str: - """Raise exception for unlogged users""" - tok = get_client_token(authorization, jwt_encryption_key) - assert tok - return tok["role"] - - -def get_account_id_or_none( - authorization: str, jwt_encryption_key: str -) -> Optional[str]: - """Returns None for unlogged users""" +def generate_random_intuid(collector_id: str) -> int: try: - tok = get_client_token(authorization, jwt_encryption_key) - return tok["account_id"] - except: - return None - - -def get_account_id_or_raise(authorization: str, jwt_encryption_key: str) -> str: - """Raise exception for unlogged users""" - tok = get_client_token(authorization, jwt_encryption_key) - return tok["account_id"] + collector_id = int(collector_id) + except ValueError: + collector_id = 0 + randint = int.from_bytes(urandom(4), byteorder) + return randint * 100 + collector_id diff --git a/ooniapi/services/ooniauth/src/ooniauth/routers/v1.py b/ooniapi/services/ooniauth/src/ooniauth/routers/v1.py index f6557296..a4b9cf9b 100644 --- a/ooniapi/services/ooniauth/src/ooniauth/routers/v1.py +++ b/ooniapi/services/ooniauth/src/ooniauth/routers/v1.py @@ -6,7 +6,8 @@ import jwt from fastapi import APIRouter, Depends, Query, HTTPException, Header, Path -from pydantic import Field, validator +from pydantic import Field +from pydantic.functional_validators import field_validator from pydantic import EmailStr from typing_extensions import Annotated @@ -22,7 +23,7 @@ from ..common.dependencies import get_settings, role_required from ..common.config import Settings from ..common.routers import BaseModel -from ..common.utils import ( +from ..common.auth import ( create_jwt, decode_jwt, get_client_token, @@ -42,7 +43,7 @@ class UserRegister(BaseModel): ) redirect_to: str = Field(title="redirect to this URL") - @validator("redirect_to") + @field_validator("redirect_to") def validate_redirect_to(cls, v): u = urlparse(v) if u.scheme != "https": diff --git a/ooniapi/services/ooniauth/src/ooniauth/routers/v2.py b/ooniapi/services/ooniauth/src/ooniauth/routers/v2.py index d5493e81..4ea9f922 100644 --- a/ooniapi/services/ooniauth/src/ooniauth/routers/v2.py +++ b/ooniapi/services/ooniauth/src/ooniauth/routers/v2.py @@ -6,7 +6,8 @@ import jwt from fastapi import APIRouter, Depends, HTTPException, Header -from pydantic import Field, validator +from pydantic import Field +from pydantic.functional_validators import field_validator from pydantic import EmailStr from typing_extensions import Annotated @@ -23,7 +24,7 @@ from ..common.dependencies import get_settings from ..common.config import Settings from ..common.routers import BaseModel -from ..common.utils import ( +from ..common.auth import ( create_jwt, decode_jwt, get_client_token, @@ -43,7 +44,7 @@ class CreateUserLogin(BaseModel): ) redirect_to: str = Field(title="redirect to this URL") - @validator("redirect_to") + @field_validator("redirect_to") def validate_redirect_to(cls, v): u = urlparse(v) if u.scheme != "https": diff --git a/ooniapi/services/ooniauth/src/ooniauth/utils.py b/ooniapi/services/ooniauth/src/ooniauth/utils.py index 2cee635c..7287b54e 100644 --- a/ooniapi/services/ooniauth/src/ooniauth/utils.py +++ b/ooniapi/services/ooniauth/src/ooniauth/utils.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from .common.utils import create_jwt +from .common.auth import create_jwt VALID_REDIRECT_TO_FQDN = ( "explorer.ooni.org", diff --git a/ooniapi/services/ooniauth/tests/test_auth_v1.py b/ooniapi/services/ooniauth/tests/test_auth_v1.py index f5f3d749..63aaa59b 100644 --- a/ooniapi/services/ooniauth/tests/test_auth_v1.py +++ b/ooniapi/services/ooniauth/tests/test_auth_v1.py @@ -1,6 +1,4 @@ from urllib.parse import parse_qs, urlparse -from ooniauth.common.utils import decode_jwt -from ooniauth.main import app from freezegun import freeze_time from html.parser import HTMLParser diff --git a/ooniapi/services/ooniauth/tests/test_auth_v2.py b/ooniapi/services/ooniauth/tests/test_auth_v2.py index 5c39cb93..813f4b03 100644 --- a/ooniapi/services/ooniauth/tests/test_auth_v2.py +++ b/ooniapi/services/ooniauth/tests/test_auth_v2.py @@ -1,6 +1,4 @@ from urllib.parse import parse_qs, urlparse -from ooniauth.common.utils import decode_jwt -from ooniauth.main import app from freezegun import freeze_time diff --git a/ooniapi/services/oonifindings/.dockerignore b/ooniapi/services/oonifindings/.dockerignore new file mode 100644 index 00000000..4f7a82b5 --- /dev/null +++ b/ooniapi/services/oonifindings/.dockerignore @@ -0,0 +1,10 @@ +.DS_Store +*.log +*.pyc +*.swp +*.env +.coverage +coverage.xml +dist/ +.venv/ +__pycache__/ diff --git a/ooniapi/services/oonifindings/.gitignore b/ooniapi/services/oonifindings/.gitignore new file mode 100644 index 00000000..9a1b4f54 --- /dev/null +++ b/ooniapi/services/oonifindings/.gitignore @@ -0,0 +1,3 @@ +/dist +/coverage_html +*.coverage* diff --git a/ooniapi/services/oonifindings/Dockerfile b/ooniapi/services/oonifindings/Dockerfile new file mode 100644 index 00000000..a15c08f4 --- /dev/null +++ b/ooniapi/services/oonifindings/Dockerfile @@ -0,0 +1,29 @@ +# Python builder +FROM python:3.11-bookworm as builder +ARG BUILD_LABEL=docker + +WORKDIR /build + +RUN python -m pip install hatch + +COPY . /build + +# When you build stuff on macOS you end up with ._ files +# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them +RUN find /build -type f -name '._*' -delete + +RUN echo "$BUILD_LABEL" > /build/src/oonifindings/BUILD_LABEL + +RUN hatch build + +### Actual image running on the host +FROM python:3.11-bookworm as runner + +WORKDIR /app + +COPY --from=builder /build/README.md /app/ +COPY --from=builder /build/dist/*.whl /app/ +RUN pip install /app/*whl && rm /app/*whl + +CMD ["uvicorn", "oonifindings.main:app", "--host", "0.0.0.0", "--port", "80"] +EXPOSE 80 diff --git a/ooniapi/services/oonifindings/LICENSE.txt b/ooniapi/services/oonifindings/LICENSE.txt new file mode 100644 index 00000000..3ec29c80 --- /dev/null +++ b/ooniapi/services/oonifindings/LICENSE.txt @@ -0,0 +1,26 @@ +Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ooniapi/services/oonifindings/Makefile b/ooniapi/services/oonifindings/Makefile new file mode 100644 index 00000000..53fc7db3 --- /dev/null +++ b/ooniapi/services/oonifindings/Makefile @@ -0,0 +1,64 @@ +SERVICE_NAME ?= oonifindings + +ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME) +IMAGE_NAME ?= ooni/api-$(SERVICE_NAME) +DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))") +GIT_FULL_SHA ?= $(shell git rev-parse HEAD) +SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8) +PKG_VERSION := $(shell hatch version) + +BUILD_LABEL := $(DATE)-$(SHORT_SHA) +VERSION_LABEL = v$(PKG_VERSION) +ENV_LABEL ?= latest + +print-labels: + echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}" + echo "PKG_VERSION=${PKG_VERSION}" + echo "BUILD_LABEL=${BUILD_LABEL}" + echo "VERSION_LABEL=${VERSION_LABEL}" + echo "ENV_LABEL=${ENV_LABEL}" + +init: + hatch env create + +docker-build: + # We need to use tar -czh to resolve the common dir symlink + tar -czh . | docker build \ + --build-arg BUILD_LABEL=${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${VERSION_LABEL} \ + -t ${IMAGE_NAME}:${ENV_LABEL} \ + - + echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})" + +docker-push: + # We need to use tar -czh to resolve the common dir symlink + docker push ${IMAGE_NAME}:${BUILD_LABEL} + docker push ${IMAGE_NAME}:${VERSION_LABEL} + docker push ${IMAGE_NAME}:${ENV_LABEL} + +docker-smoketest: + ./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL} + +imagedefinitions.json: + echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json + +test: + hatch run test + +test-cov: + hatch run test-cov + +build: + hatch build + +clean: + hatch clean + rm -f imagedefinitions.json + rm -rf build dist *eggs *.egg-info + rm -rf .venv + +run: + hatch run uvicorn $(SERVICE_NAME).main:app + +.PHONY: init test build clean docker print-labels diff --git a/ooniapi/services/oonifindings/README.md b/ooniapi/services/oonifindings/README.md new file mode 100644 index 00000000..1e370a82 --- /dev/null +++ b/ooniapi/services/oonifindings/README.md @@ -0,0 +1,21 @@ +# oonifindings + +[![PyPI - Version](https://img.shields.io/pypi/v/oonifindings.svg)](https://pypi.org/project/oonifindings) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/oonifindings.svg)](https://pypi.org/project/oonifindings) + +----- + +**Table of Contents** + +- [Installation](#installation) +- [License](#license) + +## Installation + +```console +pip install oonifindings +``` + +## License + +`oonifindings` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license. diff --git a/ooniapi/services/oonifindings/buildspec.yml b/ooniapi/services/oonifindings/buildspec.yml new file mode 100644 index 00000000..0d6cb126 --- /dev/null +++ b/ooniapi/services/oonifindings/buildspec.yml @@ -0,0 +1,29 @@ +version: 0.2 +env: + variables: + OONI_CODE_PATH: ooniapi/services/oonifindings + DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token + +phases: + install: + runtime-versions: + python: 3.11 + + pre_build: + commands: + - echo "Logging in to dockerhub" + - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text) + - echo $DOCKER_SECRET | docker login --username ooni --password-stdin + + build: + commands: + - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION} + - cd $OONI_CODE_PATH + - make docker-build + - make docker-smoketest + - make docker-push + - make imagedefinitions.json + - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json + +artifacts: + files: imagedefinitions.json diff --git a/ooniapi/services/oonifindings/migrations/clickhouse_init_tables.sql b/ooniapi/services/oonifindings/migrations/clickhouse_init_tables.sql new file mode 100644 index 00000000..8216cc7f --- /dev/null +++ b/ooniapi/services/oonifindings/migrations/clickhouse_init_tables.sql @@ -0,0 +1,28 @@ +-- Create tables for integration tests + +CREATE TABLE default.incidents +( + `id` String, + `title` String, + `short_description` String, + `text` String, + `start_time` Datetime DEFAULT now(), + `end_time` Nullable(Datetime), + `create_time` Datetime, + `update_time` Datetime DEFAULT now(), + `creator_account_id` FixedString(32), + `reported_by` String, + `email_address` String, + `event_type` LowCardinality(String), + `published` UInt8, + `deleted` UInt8 DEFAULT 0, + `CCs` Array(FixedString(2)), + `ASNs` Array(String), + `domains` Array(String), + `tags` Array(String), + `links` Array(String), + `test_names` Array(String), +) +ENGINE=ReplacingMergeTree +ORDER BY (create_time, creator_account_id, id) +SETTINGS index_granularity = 8192; diff --git a/ooniapi/services/oonifindings/pyproject.toml b/ooniapi/services/oonifindings/pyproject.toml new file mode 100644 index 00000000..ccfeff33 --- /dev/null +++ b/ooniapi/services/oonifindings/pyproject.toml @@ -0,0 +1,106 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "oonifindings" +dynamic = ["version"] +description = '' + +dependencies = [ + "fastapi ~= 0.108.0", + "clickhouse-driver ~= 0.2.6", + "sqlalchemy ~= 2.0.27", + "pydantic-settings ~= 2.1.0", + "uvicorn ~= 0.25.0", + "statsd ~= 4.0.1", + "uvicorn ~= 0.25.0", + "httpx ~= 0.26.0", + "pyjwt ~= 2.8.0", + "prometheus-fastapi-instrumentator ~= 6.1.0", + "prometheus-client", +] + +readme = "README.md" +requires-python = ">=3.11" +license = "BSD-3-Clause" +keywords = [] +authors = [ + { name = "OONI", email = "contact@ooni.org" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[project.urls] +Documentation = "https://docs.ooni.org" +Issues = "https://github.com/ooni/backend/issues" +Source = "https://github.com/ooni/backend" + +[tool.hatch.version] +path = "src/oonifindings/__about__.py" + +[tool.hatch.build.targets.sdist] +include = ["BUILD_LABEL"] + +[tool.hatch.build.targets.wheel] +packages = ["src/oonifindings"] +artifacts = ["BUILD_LABEL"] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.envs.default] +dependencies = [ + "pytest", + "pytest-cov", + "click", + "black", + "pytest-asyncio", + "pytest-docker", +] +path = ".venv/" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] + +[[tool.hatch.envs.all.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] + +[tool.hatch.envs.types] +dependencies = [ + "mypy>=1.0.0", +] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/oonifindings tests}" + +[tool.coverage.run] +source_pkgs = ["oonifindings", "tests"] +branch = true +parallel = true +omit = [ + "src/oonifindings/common/*", + "src/oonifindings/__about__.py" +] + +[tool.coverage.paths] +oonifindings = ["src/oonifindings", "*/oonifindings/src/oonifindings"] +tests = ["tests", "*/oonifindings/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/ooniapi/services/oonifindings/scripts/docker-smoketest.sh b/ooniapi/services/oonifindings/scripts/docker-smoketest.sh new file mode 100644 index 00000000..fc17b55e --- /dev/null +++ b/ooniapi/services/oonifindings/scripts/docker-smoketest.sh @@ -0,0 +1,34 @@ +h + +set -ex + +if [ $# -eq 0 ]; then + echo "Error: No Docker image name provided." + echo "Usage: $0 [IMAGE_NAME]" + exit 1 +fi + +IMAGE=$1 +CONTAINER_NAME=ooniapi-smoketest-$RANDOM +PORT=$((RANDOM % 10001 + 30000)) + +cleanup() { + echo "cleaning up" + docker logs $CONTAINER_NAME + docker stop $CONTAINER_NAME >/dev/null 2>&1 + docker rm $CONTAINER_NAME >/dev/null 2>&1 +} + +echo "[+] Running smoketest of ${IMAGE}" +docker run -d --name $CONTAINER_NAME -p $PORT:80 ${IMAGE} + +trap cleanup INT TERM EXIT + +sleep 2 +response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/health) +if [ "${response}" -eq 200 ]; then + echo "Smoke test passed: Received 200 OK from /health endpoint." +else + echo "Smoke test failed: Did not receive 200 OK from /health endpoint. Received: $response" + exit 1 +fi diff --git a/ooniapi/services/oonifindings/src/oonifindings/__about__.py b/ooniapi/services/oonifindings/src/oonifindings/__about__.py new file mode 100644 index 00000000..a75e15d4 --- /dev/null +++ b/ooniapi/services/oonifindings/src/oonifindings/__about__.py @@ -0,0 +1,2 @@ +# TODO(decfox): set VERSION here +VERSION = "0.0.1" diff --git a/ooniapi/services/oonifindings/src/oonifindings/__init__.py b/ooniapi/services/oonifindings/src/oonifindings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ooniapi/services/oonifindings/src/oonifindings/common b/ooniapi/services/oonifindings/src/oonifindings/common new file mode 120000 index 00000000..3f599f25 --- /dev/null +++ b/ooniapi/services/oonifindings/src/oonifindings/common @@ -0,0 +1 @@ +../../../../common/src/common \ No newline at end of file diff --git a/ooniapi/services/oonifindings/src/oonifindings/dependencies.py b/ooniapi/services/oonifindings/src/oonifindings/dependencies.py new file mode 100644 index 00000000..f3d86738 --- /dev/null +++ b/ooniapi/services/oonifindings/src/oonifindings/dependencies.py @@ -0,0 +1,15 @@ +from typing import Annotated + +from fastapi import Depends + +from clickhouse_driver import Client as Clickhouse + +from .common.config import Settings +from .common.dependencies import get_settings + +def get_clickhouse_session(settings: Annotated[Settings, Depends(get_settings)]): + db = Clickhouse.from_url(settings.clickhouse_url) + try: + yield db + finally: + db.disconnect() diff --git a/ooniapi/services/oonifindings/src/oonifindings/main.py b/ooniapi/services/oonifindings/src/oonifindings/main.py new file mode 100644 index 00000000..3f003c8a --- /dev/null +++ b/ooniapi/services/oonifindings/src/oonifindings/main.py @@ -0,0 +1,110 @@ +import logging +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware + +from pydantic import BaseModel + +from prometheus_fastapi_instrumentator import Instrumentator + +from .routers import v1 + +from .dependencies import get_settings, get_clickhouse_session +from .common.version import get_build_label, get_pkg_version +from .common.clickhouse_utils import query_click +from .common.metrics import mount_metrics + + +pkg_name = "oonifindings" + +pkg_version = get_pkg_version(pkg_name) +build_label = get_build_label(pkg_name) + +log = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = get_settings() + logging.basicConfig(level=getattr(logging, settings.log_level.upper())) + mount_metrics(app, instrumentor.registry) + yield + + +app = FastAPI(lifespan=lifespan) + +instrumentor = Instrumentator().instrument( + app, metric_namespace="ooniapi", metric_subsystem="oonifindings" +) + +# TODO: temporarily enable all +origins = ["*"] +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(v1.router, prefix="/api") + + +@app.get("/version") +async def version(): + return {"version": pkg_version, "build_label": build_label} + + +class HealthStatus(BaseModel): + status: str + errors: list[str] = [] + version: str + build_label: str + + +# TODO(decfox): Add minimal health check functionality +@app.get("/health") +async def health( + settings=Depends(get_settings), + db=Depends(get_clickhouse_session), +): + errors = [] + + try: + query = f"""SELECT id, update_time, start_time, end_time, reported_by, + title, event_type, published, CCs, ASNs, domains, tags, test_names, + links, short_description, email_address, create_time, creator_account_id + FROM incidents FINAL + """ + query_click(db=db, query=query, query_params={}) + except Exception as exc: + log.error(exc) + errors.append("db_error") + + if settings.jwt_encryption_key == "CHANGEME": + err = "bad_jwt_secret" + log.error(err) + errors.append(err) + + if settings.prometheus_metrics_password == "CHANGEME": + err = "bad_prometheus_password" + log.error(err) + errors.append(err) + + if len(errors) > 0: + raise HTTPException(status_code=400, detail="health check failed") + + status = "ok" + + return { + "status": status, + "errors": errors, + "version": pkg_version, + "build_label": build_label, + } + + +@app.get("/") +async def root(): + return {"message": "Hello OONItarian"} diff --git a/ooniapi/services/oonifindings/src/oonifindings/routers/v1.py b/ooniapi/services/oonifindings/src/oonifindings/routers/v1.py new file mode 100644 index 00000000..301af35c --- /dev/null +++ b/ooniapi/services/oonifindings/src/oonifindings/routers/v1.py @@ -0,0 +1,479 @@ +""" +OONIFindings incidents management +""" + +from datetime import datetime, timezone +from typing import List, Dict, Optional, Union, Tuple, Any, Annotated +import logging + +from clickhouse_driver import Client as Clickhouse +from fastapi import APIRouter, Depends, Header, Response, HTTPException, Query + +from pydantic import Field +from pydantic.functional_validators import field_validator + +from ..common.routers import BaseModel, ISO_FORMAT_DATE +from ..common.dependencies import get_settings, role_required +from ..common.auth import ( + check_email_address, + get_account_id_or_raise, + get_account_id_or_none, + get_client_role +) +from ..common.utils import setnocacheresponse, generate_random_intuid +from ..common.clickhouse_utils import query_click, raw_query, insert_click, optimize_table +from ..dependencies import get_clickhouse_session + +log = logging.getLogger(__name__) + +router = APIRouter() + + +def utcnow_seconds(): + return datetime.now(timezone.utc).replace(microsecond=0) + + +class OONIFindingId(BaseModel): + incident_id: str = Field( + alias="id" + ) + +class OONIFindingWithMail(OONIFindingId): + email_address: str = Field( + default="", title="email address of user" + ) + + +class OONIFinding(OONIFindingWithMail): + title: str = Field( + title="title of the ooni finding" + ) + short_description: str = Field( + default="", title="short description of the oonifinding report" + ) + start_time: datetime = Field( + title="date when the oonifinding incident started" + ) + create_time: Optional[datetime] = Field( + default=None, title="date when the oonifinding report was created" + ) + update_time: Optional[datetime] = Field( + default=None, title="time when the oonifinding report was last updated" + ) + end_time: Optional[datetime] = Field( + default=None, title="time when the oonifinding incident ended" + ) + reported_by: str = Field( + default="", title="name of the oonifinding reporter" + ) + creator_account_id: Optional[str] = Field( + default="", title="account id of the oonifinding report creator" + ) + published: bool = Field( + default=False, title="binary check if event is published" + ) + event_type: str = Field( + default="", title="type of oonifinding event" + ) + ASNs: List[int] = Field( + default=[], description="list of ASNs associate with the oonifinding" + ) + CCs: List[str] = Field( + default=[], description="list of country codes associated with the oonifinding" + ) + tags: List[str] = Field( + default=[], description="tags associated with the oonifinding" + ) + test_names: List[str] = Field( + default=[], description="ooni tests associated with the oonifinding" + ) + domains: List[str] = Field( + default=[], description="list of domains associated with the oonifinding" + ) + links: List[str] = Field( + default=[], description="links associated with the oonifinding" + ) + mine: Optional[bool] = Field( + default=False, title="check if creator account id matches user" + ) + + +class OONIFindingWithText(OONIFinding): + text: str = Field( + title="content of the oonifinding report" + ) + + @field_validator("title", "text") + @classmethod + def check_empty(cls, v: str): + if not v: + raise ValueError("field cannot be empty") + return v + + +class OONIFindingIncident(BaseModel): + incident: OONIFindingWithText + + +class OONIFindingIncidents(BaseModel): + incidents: List[OONIFinding] + + +@router.get( + "/v1/incidents/search", + tags=["oonifindings"], + response_model = OONIFindingIncidents +) +def list_oonifindings( + only_mine: Annotated[ + bool, + Query(description="show only owned items") + ], + response: Response, + authorization: str = Header("authorization"), + db=Depends(get_clickhouse_session), + settings=Depends(get_settings), +): + """ + Search and list incidents + """ + log.debug("listing incidents") + where = "WHERE deleted != 1" + query_params = {} + + account_id = get_account_id_or_none( + authorization, jwt_encryption_key=settings.jwt_encryption_key + ) + if only_mine: + if account_id is None: + return OONIFindingIncidents(incidents=[]) + where += "\nAND creator_account_id = %(account_id)s" + + if account_id is None: + # non-published incidents are not exposed to anon users + where += "\nAND published = 1" + query_params["account_id"] = "never-match" + else: + query_params["account_id"] = account_id + + query = f"""SELECT id, update_time, start_time, end_time, reported_by, + title, event_type, published, CCs, ASNs, domains, tags, test_names, + links, short_description, email_address, create_time, + creator_account_id = %(account_id)s as mine + FROM incidents FINAL + {where} + ORDER BY title + """ + q = query_click(db=db, query=query, query_params=query_params) + + incidents = list(q) + client_role = get_client_role(authorization, jwt_encryption_key=settings.jwt_encryption_key) + for incident in incidents: + incident["published"] = bool(incident["published"]) + if account_id is None or client_role != "admin": + incident["email_address"] = "" + + setnocacheresponse(response) + incident_models = [] + for i in range(len(incidents)): + incident = incidents[i] + incident_model = OONIFinding.model_validate(incident) + incident_models.append(incident_model) + return OONIFindingIncidents(incidents=incident_models) + + +@router.get( + "/v1/incidents/show/{incident_id}", + tags=["oonifindings"], + response_model=OONIFindingIncident +) +def get_oonifinding_by_id( + incident_id: str, + response: Response, + authorization: str = Header("authorization"), + db=Depends(get_clickhouse_session), + settings=Depends(get_settings) +): + """ + Returns an incident + """ + log.debug("showing incident") + where = "WHERE id = %(id)s AND deleted != 1" + account_id = get_account_id_or_none( + authorization, jwt_encryption_key=settings.jwt_encryption_key + ) + if account_id is None: + # non-published incidents are not exposed to anon users + where += "\nAND published = 1" + query_params = {"id": incident_id, "account_id": "never-match"} + else: + query_params = {"id": incident_id, "account_id": account_id} + + query = f"""SELECT id, update_time, start_time, end_time, reported_by, + title, text, event_type, published, CCs, ASNs, domains, tags, test_names, + links, short_description, email_address, create_time, + creator_account_id = %(account_id)s AS mine + FROM incidents FINAL + {where} + LIMIT 1 + """ + q = query_click(db=db, query=query, query_params=query_params) + if len(q) < 1: + raise HTTPException(status_code=404, detail="Incident not found") + + incident = q[0] + incident["published"] = bool(incident["published"]) + client_role = get_client_role(authorization, jwt_encryption_key=settings.jwt_encryption_key) + if account_id is None or client_role != "admin": + incident["email_address"] = "" # hide email + + # TODO: cache if possible + setnocacheresponse(response) + incident_model = OONIFindingWithText.model_validate(incident) + return OONIFindingIncident(incident=incident_model) + + +def prepare_incident_dict(incident: OONIFinding) -> Dict: + incident.start_time = incident.start_time.replace(microsecond=0) + if incident.end_time is not None: + incident.end_time = incident.end_time.replace(microsecond=0) + delta = incident.end_time - incident.start_time + if delta.total_seconds() < 0: + raise HTTPException(status_code=400, detail="invalid query paramters") + incident_dict = incident.model_dump(by_alias=True) + return incident_dict + + +def user_cannot_update( + db: Clickhouse, + authorization: str, + jwt_encryption_key: str, + incident_id: str, +) -> bool: + # Check if there is already an incident and belongs to a different user + query = """SELECT count() AS cnt + FROM incidents FINAL + WHERE deleted != 1 + AND id = %(incident_id)s + AND creator_account_id != %(account_id)s + """ + account_id = get_account_id_or_raise(authorization, jwt_encryption_key=jwt_encryption_key) + query_params = dict(incident_id=incident_id, account_id=account_id) + q = query_click(db, query, query_params) + return q[0]["cnt"] > 0 + + +def verify_user( + db: Clickhouse, + authorization: str, + jwt_encryption_key: str, + incident_id: str, + email_address: str, + key: str +): + if user_cannot_update( + db, authorization, jwt_encryption_key=jwt_encryption_key, incident_id=incident_id + ): + raise HTTPException(status_code=400, detail="Attempted to create, update or delete an item belonging to another user") + + if not check_email_address( + authorization=authorization, + jwt_encryption_key=jwt_encryption_key, + email_address=email_address, + key=key + ): + raise HTTPException(status_code=400, detail="Invalid email address for owner account") + + +class OONIFindingCreateUpdate(OONIFindingWithText): + pass + + +class OONIFindingsUpdateResponse(OONIFindingId): + r: Union[int, Tuple[List[Any]]] = Field( + default=0, title="result of the update operation" + ) + + +@router.post( + "/v1/incidents/create", + dependencies=[Depends(role_required(["admin"]))], + tags=["oonifindings"], + response_model=OONIFindingsUpdateResponse +) +def create_oonifinding( + create_request: OONIFindingCreateUpdate, + response: Response, + authorization: str = Header("authorization"), + db=Depends(get_clickhouse_session), + settings=Depends(get_settings) +): + """ + Create an incident + """ + if not check_email_address( + authorization=authorization, + jwt_encryption_key=settings.jwt_encryption_key, + email_address=create_request.email_address, + key=settings.account_id_hashing_key + ): + raise HTTPException(status_code=400, detail="Invalid email address for creator account") + + # assert create_request + if create_request.published: + raise HTTPException(status_code=400, detail="Invalid publish parameter on create request") + + + incident_id = str(generate_random_intuid(collector_id=settings.collector_id)) + create_request.incident_id = incident_id + create_request.create_time = utcnow_seconds() + create_request.creator_account_id = get_account_id_or_raise( + authorization, jwt_encryption_key=settings.jwt_encryption_key + ) + incident_dict = prepare_incident_dict(incident=create_request) + + log.info(f"Creating incident {incident_id}") + + query = """INSERT INTO incidents + (id, start_time, end_time, creator_account_id, reported_by, title, + text, event_type, published, CCs, ASNs, domains, tags, links, + test_names, short_description, email_address, create_time) + VALUES + """ + r = insert_click(db, query, [incident_dict]) + optimize_table(db, tblname="incidents") + + setnocacheresponse(response) + return OONIFindingsUpdateResponse(r=r, id=incident_id) + + +@router.post( + "/v1/incidents/update", + dependencies=[Depends(role_required(["admin", "user"]))], + tags=["oonifindings"], + response_model=OONIFindingsUpdateResponse +) +def update_oonifinding( + update_request: OONIFindingCreateUpdate, + response: Response, + authorization: str = Header("authorization"), + db=Depends(get_clickhouse_session), + token=Depends(role_required(["admin", "user"])), + settings=Depends(get_settings) +): + """ + Update an incident + """ + incident_id = update_request.incident_id + if token["role"] != "admin": + verify_user( + db, + authorization=authorization, + jwt_encryption_key=settings.jwt_encryption_key, + incident_id=incident_id, + email_address=update_request.email_address, + key=settings.account_id_hashing_key, + ) + + if update_request.published is True: + raise HTTPException(status_code=400, detail="Not enough permissions to publish") + + update_request.creator_account_id = get_account_id_or_raise( + authorization, jwt_encryption_key=settings.jwt_encryption_key + ) + incident_dict = prepare_incident_dict(update_request) + + log.info(f"Updating incident {incident_id}") + + insert_query = """INSERT INTO incidents + (id, start_time, end_time, creator_account_id, reported_by, title, + text, event_type, published, CCs, ASNs, domains, tags, links, + test_names, short_description, email_address, create_time) + VALUES + """ + r = insert_click(db, insert_query, [incident_dict]) + log.debug(f"Result: {r}") + optimize_table(db, tblname="incidents") + + setnocacheresponse(response) + return OONIFindingsUpdateResponse(r=r, id=incident_id) + + +@router.post( + "/v1/incidents/delete", + tags=["oonifindings"], +) +def delete_oonifinding( + delete_request: OONIFindingWithMail, + response: Response, + authorization: str = Header("authorization"), + token=Depends(role_required(["admin", "user"])), + db=Depends(get_clickhouse_session), + settings=Depends(get_settings) +): + """ + Delete an incident + """ + assert delete_request + incident_id = delete_request.incident_id + if token["role"] != "admin": + try: + verify_user( + db, + authorization=authorization, + jwt_encryption_key=settings.jwt_encryption_key, + incident_id=incident_id, + email_address=delete_request.email_address, + key=settings.account_id_hashing_key, + ) + except: + raise + + query = "ALTER TABLE incidents DELETE WHERE id = %(incident_id)s" + r = raw_query(db, query, {"incident_id": incident_id}) + optimize_table(db, "incidents") + setnocacheresponse(response) + return {} + + + +@router.post( + "/v1/incidents/{action}", + tags=["oonifindings"], + dependencies=[Depends(role_required(["admin"]))], + response_model=OONIFindingsUpdateResponse +) +def update_oonifinding_publish_status( + action: str, + publish_request: OONIFindingCreateUpdate, + response: Response, + db=Depends(get_clickhouse_session), +): + """ + Publish/Unpublish an incident. + """ + if action not in ("publish", "unpublish"): + raise HTTPException(status_code=400, detail="Invalid query action") + + assert publish_request + incident_id = publish_request.incident_id + + query = "SELECT * FROM incidents FINAL WHERE id = %(incident_id)s" + q = query_click(db, query, {"incident_id": incident_id}) + if len(q) < 1: + raise HTTPException(status_code=404, detail="Incident not found") + incident_dict = q[0] + incident_dict["published"] = bool(action == "publish") + + insert_query = """INSERT INTO incidents + (id, start_time, end_time, creator_account_id, reported_by, title, + text, event_type, published, CCs, ASNs, domains, tags, links, + test_names, short_description, email_address, create_time) + VALUES + """ + r = insert_click(db, insert_query, [incident_dict]) + log.debug(f"Result: {r}") + optimize_table(db, tblname="incidents") + + setnocacheresponse(response) + return OONIFindingsUpdateResponse(r=r, id=incident_id) diff --git a/ooniapi/services/oonifindings/tests/__init__.py b/ooniapi/services/oonifindings/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ooniapi/services/oonifindings/tests/conftest.py b/ooniapi/services/oonifindings/tests/conftest.py new file mode 100644 index 00000000..31dbbbd5 --- /dev/null +++ b/ooniapi/services/oonifindings/tests/conftest.py @@ -0,0 +1,139 @@ +from pathlib import Path +import pytest + +import time +import jwt + +from fastapi.testclient import TestClient +from clickhouse_driver import Client as ClickhouseClient + +from oonifindings.common.config import Settings +from oonifindings.common.auth import hash_email_address +from oonifindings.common.dependencies import get_settings +from oonifindings.main import app + +THIS_DIR = Path(__file__).parent.resolve() + + +def is_clickhouse_running(url): + try: + with ClickhouseClient.from_url(url) as client: + client.execute("SELECT 1") + return True + except Exception: + return False + + +@pytest.fixture(scope="session") +def clickhouse_server(docker_ip, docker_services): + port = docker_services.port_for("clickhouse", 9000) + url = "clickhouse://{}:{}".format(docker_ip, port) + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_clickhouse_running(url) + ) + yield url + + +def run_migration(path: Path, click: ClickhouseClient): + sql_no_comment = "\n".join( + filter(lambda x: not x.startswith("--"), path.read_text().split("\n")) + ) + queries = sql_no_comment.split(";") + for q in queries: + q = q.strip() + if not q: + continue + click.execute(q) + + +def create_db_for_fixture(conn_url): + try: + with ClickhouseClient.from_url(conn_url) as client: + migrations_dir = THIS_DIR / "migrations" + for fn in migrations_dir.iterdir(): + migration_path = fn.resolve() + run_migration(migration_path, click=client) + return conn_url + except Exception: + pytest.skip("database migration failed") + + +@pytest.fixture(scope="session") +def db(clickhouse_server): + yield create_db_for_fixture(conn_url=clickhouse_server) + + +def make_override_get_settings(**kw): + def override_get_settings(): + return Settings(**kw) + + return override_get_settings + + +@pytest.fixture +def client_with_bad_settings(): + app.dependency_overrides[get_settings] = make_override_get_settings( + clickhouse_url = "clickhouse://badhost:9000" + ) + + client = TestClient(app) + yield client + + +@pytest.fixture +def client(db): + app.dependency_overrides[get_settings] = make_override_get_settings( + clickhouse_url=db, + jwt_encryption_key="super_secure", + prometheus_metrics_password="super_secure", + account_id_hashing_key="super_secure" + ) + + client = TestClient(app) + yield client + + +def create_jwt(payload: dict) -> str: + return jwt.encode(payload, "super_secure", algorithm="HS256") + + +def create_session_token(account_id: str, role: str) -> str: + now = int(time.time()) + payload = { + "nbf": now, + "iat": now, + "exp": now + 10 * 86400, + "aud": "user_auth", + "account_id": account_id, + "login_time": None, + "role": role, + } + return create_jwt(payload) + +@pytest.fixture +def client_with_user_role(client): + client = TestClient(app) + jwt_token = create_session_token("0" * 16, "user") + client.headers = {"Authorization": f"Bearer {jwt_token}"} + yield client + + +@pytest.fixture +def client_with_admin_role(client): + client = TestClient(app) + jwt_token = create_session_token("0" * 16, "admin") + client.headers = {"Authorization": f"Bearer {jwt_token}"} + yield client + + +@pytest.fixture +def client_with_hashed_email(client): + + def _hashed_email(email: str, role: str): + client = TestClient(app) + account_id = hash_email_address(email, "super_secure") + jwt_token = create_session_token(account_id, role) + client.headers = {"Authorization": f"Bearer {jwt_token}"} + return client + + return _hashed_email diff --git a/ooniapi/services/oonifindings/tests/docker-compose.yml b/ooniapi/services/oonifindings/tests/docker-compose.yml new file mode 100644 index 00000000..7546ca5b --- /dev/null +++ b/ooniapi/services/oonifindings/tests/docker-compose.yml @@ -0,0 +1,6 @@ +version: '2' +services: + clickhouse: + image: "clickhouse/clickhouse-server" + ports: + - "9000:9000" diff --git a/ooniapi/services/oonifindings/tests/migrations/clickhouse_init_tables.sql b/ooniapi/services/oonifindings/tests/migrations/clickhouse_init_tables.sql new file mode 100644 index 00000000..8216cc7f --- /dev/null +++ b/ooniapi/services/oonifindings/tests/migrations/clickhouse_init_tables.sql @@ -0,0 +1,28 @@ +-- Create tables for integration tests + +CREATE TABLE default.incidents +( + `id` String, + `title` String, + `short_description` String, + `text` String, + `start_time` Datetime DEFAULT now(), + `end_time` Nullable(Datetime), + `create_time` Datetime, + `update_time` Datetime DEFAULT now(), + `creator_account_id` FixedString(32), + `reported_by` String, + `email_address` String, + `event_type` LowCardinality(String), + `published` UInt8, + `deleted` UInt8 DEFAULT 0, + `CCs` Array(FixedString(2)), + `ASNs` Array(String), + `domains` Array(String), + `tags` Array(String), + `links` Array(String), + `test_names` Array(String), +) +ENGINE=ReplacingMergeTree +ORDER BY (create_time, creator_account_id, id) +SETTINGS index_granularity = 8192; diff --git a/ooniapi/services/oonifindings/tests/test_integration.py b/ooniapi/services/oonifindings/tests/test_integration.py new file mode 100644 index 00000000..206e7df5 --- /dev/null +++ b/ooniapi/services/oonifindings/tests/test_integration.py @@ -0,0 +1,42 @@ +import os +import time +import random + +from multiprocessing import Process + +import httpx +import pytest +import uvicorn + + +LISTEN_PORT = random.randint(30_000, 42_000) + + +@pytest.fixture +def server(clickhouse_server): + os.environ["CLICKHOUSE_URL"] = clickhouse_server + proc = Process( + target=uvicorn.run, + args=("oonifindings.main:app"), + kwargs={"host": "127.0.0.1", "port": LISTEN_PORT, "log_level": "info"}, + daemon=True, + ) + + proc.start() + # Give it as second to start + time.sleep(1) + yield + proc.kill() + # Note: coverage is not being calculated properly + # TODO(art): https://pytest-cov.readthedocs.io/en/latest/subprocess-support.html + proc.join() + + +@pytest.mark.skip("TODO(decfox): fix integration test") +def test_integration(server): + with httpx.Client(base_url=f"http://127.0.0.1:{LISTEN_PORT}") as client: + r = client.get("/version") + assert r.status_code == 200 + r = client.get("/api/v2/incidents/search") + j = r.json() + assert isinstance(j["incidents"], list) diff --git a/ooniapi/services/oonifindings/tests/test_main.py b/ooniapi/services/oonifindings/tests/test_main.py new file mode 100644 index 00000000..d173e98b --- /dev/null +++ b/ooniapi/services/oonifindings/tests/test_main.py @@ -0,0 +1,35 @@ +import pytest + +import httpx +from fastapi.testclient import TestClient +from oonifindings.main import lifespan, app + + +def test_health_good(client): + r = client.get("health") + j = r.json() + assert j["status"] == "ok", j + assert len(j["errors"]) == 0, j + + +def test_health_bad(client_with_bad_settings): + r = client_with_bad_settings.get("health") + j = r.json() + assert j["detail"] == "health check failed", j + assert r.status_code == 400 + + +def test_metrics(client): + r = client.get("/metrics") + + +@pytest.mark.asyncio +async def test_lifecycle(): + async with lifespan(app): + client = TestClient(app) + r = client.get("/metrics") + assert r.status_code == 401 + + auth = httpx.BasicAuth(username="prom", password="super_secure") + r = client.get("/metrics", auth=auth) + assert r.status_code == 200, r.text diff --git a/ooniapi/services/oonifindings/tests/test_oonifindings.py b/ooniapi/services/oonifindings/tests/test_oonifindings.py new file mode 100644 index 00000000..1256cefc --- /dev/null +++ b/ooniapi/services/oonifindings/tests/test_oonifindings.py @@ -0,0 +1,425 @@ +""" +Integration test for OONIFindings API +""" + +from copy import deepcopy +from datetime import timedelta, datetime + +from oonifindings.routers.v1 import utcnow_seconds + +sample_start_time = (utcnow_seconds() + timedelta(minutes=-1)).strftime( + "%Y-%m-%dT%H:%M:%S.%fZ" +) + +SAMPLE_EMAIL = "sample@ooni.org" + +SAMPLE_OONIFINDING = { + "id": "", + "title": "sample oonifinding", + "short_description": "sample oonifinding description", + "reported_by": "sample user", + "email_address": SAMPLE_EMAIL, + "text": "this is a sample oonifinding incident", + "published": False, + "event_type": "incident", + "start_time": sample_start_time, + "ASNs": [], + "CCs": [ + "IN", "TZ", + ], + "tags": [], + "test_names": [ + "webconnectivity", + ], + "domains": [ + "www.google.com" + ], + "links": [] +} + +EXPECTED_OONIFINDING_PUBLIC_KEYS = [ + "id", + "title", + "short_description", + "start_time", + "end_time", + "create_time", + "update_time", + "creator_account_id", + "reported_by", + "email_address", + "text", + "mine", + "published", + "event_type", + "ASNs", + "CCs", + "tags", + "test_names", + "domains", + "links", +] + + +def test_get_version(client): + r = client.get("/version") + j = r.json() + assert "version" in j + assert "build_label" in j + + +def test_get_root(client): + r = client.get("/") + assert r.status_code == 200 + + +def test_oonifinding_validation(client, client_with_user_role): + z = deepcopy(SAMPLE_OONIFINDING) + r = client_with_user_role.post("/api/v1/incidents/create", json=z) + assert r.status_code == 401, "only admins can create incidents" + + +def test_oonifinding_creator_validation(client, client_with_hashed_email): + client_with_admin_role = client_with_hashed_email(SAMPLE_EMAIL, "admin") + + z = deepcopy(SAMPLE_OONIFINDING) + + z["email_address"] = "" + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 400, "email hash does not match with account id" + + z["email_address"] = SAMPLE_EMAIL + z["title"] = "" + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 422, "empty title should be rejected" + + z["title"] = "sample oonifinding" + z["text"] = "" + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 422, "empty text should be rejected" + + z["text"] = "sample text for oonifinding incident" + start_time = datetime.strptime(sample_start_time, "%Y-%m-%dT%H:%M:%S.%fZ") + sample_end_time = start_time + timedelta(minutes=-1) + z["end_time"] = sample_end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 400, "invalid end_time should be rejected" + + sample_end_time = start_time + timedelta(minutes=1) + z["end_time"] = sample_end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.headers["Cache-Control"] == "no-cache" + + +def test_oonifinding_publish(client, client_with_hashed_email): + client_with_admin_role = client_with_hashed_email(SAMPLE_EMAIL, "admin") + client_with_user_role = client_with_hashed_email(SAMPLE_EMAIL, "user") + + z = deepcopy(SAMPLE_OONIFINDING) + + z["published"] = True + r = client_with_admin_role.post("/api/v1/incidents/create", json=z) + assert r.status_code == 400, "published true should be rejected" + + z["published"] = False + r = client_with_admin_role.post("/api/v1/incidents/create", json=z) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.headers["Cache-Control"] == "no-cache" + + incident_id = r.json()["id"] + assert incident_id + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident_payload = r.json()["incident"] + + r = client_with_admin_role.post("api/v1/incidents/random", json=incident_payload) + assert r.status_code == 400, "only publish and unpublish are valid supported actions" + + r = client_with_user_role.post("api/v1/incidents/publish", json=incident_payload) + assert r.status_code == 401, "only admins can publish incidents" + + incident_payload["id"] = "sample id" + r = client_with_admin_role.post("api/v1/incidents/publish", json=incident_payload) + assert r.status_code == 404, "valid incident id should be passed" + + incident_payload["id"] = incident_id + r = client_with_admin_role.post("api/v1/incidents/publish", json=incident_payload) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.json()["id"] == incident_id + assert r.headers["Cache-Control"] == "no-cache" + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident = r.json()["incident"] + assert incident + assert incident["published"] is True + + r = client_with_admin_role.post("api/v1/incidents/unpublish", json=incident_payload) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.json()["id"] == incident_id + assert r.headers["Cache-Control"] == "no-cache" + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident = r.json()["incident"] + assert incident + assert incident["published"] is False + + +def test_oonifinding_delete(client, client_with_hashed_email): + client_with_admin_role = client_with_hashed_email(SAMPLE_EMAIL, "admin") + client_with_user_role = client_with_hashed_email(SAMPLE_EMAIL, "user") + + z = deepcopy(SAMPLE_OONIFINDING) + + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.headers["Cache-Control"] == "no-cache" + + incident_id = r.json()["id"] + assert incident_id + + z["id"] = incident_id + r = client_with_admin_role.post("api/v1/incidents/delete", json=z) + assert r.status_code == 200 + assert r.headers["Cache-Control"] == "no-cache" + + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.headers["Cache-Control"] == "no-cache" + + incident_id = r.json()["id"] + assert incident_id + + z["id"] = incident_id + z["email_address"] = "" + r = client_with_user_role.post("api/v1/incidents/delete", json=z) + assert r.status_code == 400 + + z["email_address"] = SAMPLE_EMAIL + mismatched_client = client_with_hashed_email("user@ooni.org", "user") + r = mismatched_client.post("api/v1/incidents/delete", json=z) + assert r.status_code == 400 + + r = client_with_user_role.post("api/v1/incidents/delete", json=z) + assert r.status_code == 200 + assert r.headers["Cache-Control"] == "no-cache" + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + assert r.status_code == 404 + + +def test_oonifinding_update(client, client_with_hashed_email): + client_with_admin_role = client_with_hashed_email(SAMPLE_EMAIL, "admin") + client_with_user_role = client_with_hashed_email(SAMPLE_EMAIL, "user") + + z = deepcopy(SAMPLE_OONIFINDING) + + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.headers["Cache-Control"] == "no-cache" + + incident_id = r.json()["id"] + assert incident_id + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident_payload = r.json()["incident"] + + sample_replacement_text = "sample replacement text for update" + incident_payload["text"] = sample_replacement_text + r = client_with_admin_role.post("api/v1/incidents/update", json=incident_payload) + assert r.json()["r"] == 1 + assert r.json()["id"] == incident_id + assert r.headers["Cache-Control"] == "no-cache" + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident_payload = r.json()["incident"] + assert incident_payload + assert incident_payload["text"] == sample_replacement_text + + incident_payload["text"] = "" + r = client_with_admin_role.post("api/v1/incidents/update", json=incident_payload) + assert r.status_code == 422, "cannot update with empty text" + + incident_payload["text"] = sample_replacement_text + incident_payload["title"] = "" + r = client_with_admin_role.post("api/v1/incidents/update", json=incident_payload) + assert r.status_code == 422, "cannot update with empty title" + + incident_payload["title"] = z["title"] + sample_replacement_description = "sample replacement discription for update" + incident_payload["short_description"] = sample_replacement_description + + incident_payload["email_address"] = "" + r = client_with_user_role.post("api/v1/incidents/update", json=incident_payload) + assert r.status_code == 400, "cannot update with invalid email" + + incident_payload["email_address"] = SAMPLE_EMAIL + mismatched_client = client_with_hashed_email("user@ooni.org", "user") + r = mismatched_client.post("api/v1/incidents/update", json=incident_payload) + assert r.status_code == 400, "email should match account id" + + r = client_with_user_role.post("api/v1/incidents/update", json=incident_payload) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.json()["id"] == incident_id + assert r.headers["Cache-Control"] == "no-cache" + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident_payload = r.json()["incident"] + assert incident_payload + assert incident_payload["short_description"] == sample_replacement_description + + sample_tag = "sample_tag" + incident_payload["tags"].append(sample_tag) + r = client_with_user_role.post("api/v1/incidents/update", json=incident_payload) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.json()["id"] == incident_id + assert r.headers["Cache-Control"] == "no-cache" + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident_payload = r.json()["incident"] + assert incident_payload + assert len(incident_payload["tags"]) == 1 + assert incident_payload["tags"][0] == sample_tag + + incident_payload["published"] = True + r = client_with_user_role.post("api/v1/incidents/update", json=incident_payload) + assert r.status_code == 400, "user role cannot publish incident" + + r = client_with_admin_role.post("api/v1/incidents/update", json=incident_payload) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.json()["id"] == incident_id + assert r.headers["Cache-Control"] == "no-cache" + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident_payload = r.json()["incident"] + assert incident_payload + assert incident_payload["published"] == True + + +# TODO(decfox): add checks for fetched incident fields +def test_oonifinding_workflow( + client, + client_with_hashed_email, + client_with_user_role + ): + client_with_admin_role = client_with_hashed_email(SAMPLE_EMAIL, "admin") + + z = deepcopy(SAMPLE_OONIFINDING) + + r = client_with_admin_role.post("api/v1/incidents/create", json=z) + assert r.status_code == 200 + assert r.json()["r"] == 1 + assert r.headers["Cache-Control"] == "no-cache" + + incident_id = r.json()["id"] + assert incident_id + + r = client.get(f"api/v1/incidents/show/{incident_id}") + assert r.status_code == 404, "unpublished events cannot be seen with invalid account id" + + r = client_with_user_role.get(f"api/v1/incidents/show/{incident_id}") + incident = r.json()["incident"] + assert incident + assert incident["mine"] is False + assert incident["email_address"] == "" + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident = r.json()["incident"] + assert incident + assert incident["mine"] is True + assert incident["email_address"] == z["email_address"] + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + # publish incident and test + r = client_with_admin_role.post("api/v1/incidents/publish", json=incident) + assert r.json()["r"] == 1 + + r = client.get(f"api/v1/incidents/show/{incident_id}") + incident = r.json()["incident"] + assert incident + assert incident["mine"] is False + assert incident["email_address"] == "" + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + r = client_with_user_role.get(f"api/v1/incidents/show/{incident_id}") + incident = r.json()["incident"] + assert incident + assert incident["mine"] is False + assert incident["email_address"] == "" + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + r = client_with_admin_role.get(f"api/v1/incidents/show/{incident_id}") + incident = r.json()["incident"] + assert incident + assert incident["mine"] is True + assert incident["email_address"] == z["email_address"] + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + EXPECTED_OONIFINDING_PUBLIC_KEYS.remove("text") + + r = client.get("api/v1/incidents/search?only_mine=false") + assert r.status_code == 200 + incidents = r.json()["incidents"] + assert len(incidents) == 2 + for incident in incidents: + assert incident["email_address"] == "" + assert incident["mine"] is False + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + + r = client_with_user_role.get("api/v1/incidents/search?only_mine=false") + incidents = r.json()["incidents"] + assert len(incidents) == 4 + for incident in incidents: + assert incident["email_address"] == "" + assert incident["mine"] is False + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + r = client_with_admin_role.get("api/v1/incidents/search?only_mine=false") + incidents = r.json()["incidents"] + assert len(incidents) == 4 + for incident in incidents: + assert incident["email_address"] == SAMPLE_EMAIL + assert incident["mine"] is True + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + r = client.get("api/v1/incidents/search?only_mine=true") + assert r.status_code == 200 + incidents = r.json()["incidents"] + assert len(incidents) == 0 + + r = client_with_user_role.get("api/v1/incidents/search?only_mine=true") + assert r.status_code == 200 + incidents = r.json()["incidents"] + assert len(incidents) == 0 + + client_account_with_user_role = client_with_hashed_email(SAMPLE_EMAIL, "user") + + r = client_account_with_user_role.get("api/v1/incidents/search?only_mine=true") + assert r.status_code == 200 + incidents = r.json()["incidents"] + assert len(incidents) == 4 + for incident in incidents: + assert incident["email_address"] == "" + assert incident["mine"] is True + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) + + r = client_with_admin_role.get("api/v1/incidents/search?only_mine=true") + assert r.status_code == 200 + incidents = r.json()["incidents"] + assert len(incidents) == 4 + for incident in incidents: + assert incident["email_address"] == SAMPLE_EMAIL + assert incident["mine"] is True + assert sorted(incident.keys()) == sorted(EXPECTED_OONIFINDING_PUBLIC_KEYS) diff --git a/ooniapi/services/oonirun/src/oonirun/routers/v2.py b/ooniapi/services/oonirun/src/oonirun/routers/v2.py index d6b2e3f1..d7be7eed 100644 --- a/ooniapi/services/oonirun/src/oonirun/routers/v2.py +++ b/ooniapi/services/oonirun/src/oonirun/routers/v2.py @@ -4,22 +4,23 @@ https://github.com/ooni/spec/blob/master/backends/bk-005-ooni-run-v2.md """ -from datetime import datetime, timedelta, timezone, date +from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, Tuple import logging import sqlalchemy as sa from sqlalchemy.orm import Session from fastapi import APIRouter, Depends, Query, HTTPException, Header, Path -from pydantic import computed_field, Field, validator +from pydantic import computed_field, Field +from pydantic.functional_validators import field_validator from typing_extensions import Annotated from .. import models from ..common.routers import BaseModel from ..common.dependencies import get_settings, role_required -from ..common.utils import ( - get_account_id_or_none, +from ..common.auth import ( + get_account_id_or_none, ) from ..dependencies import get_postgresql_session @@ -89,8 +90,9 @@ class OONIRunLinkBase(BaseModel): description="full description of the ooni run link in different languages", ) - @validator("name_intl", "short_description_intl", "description_intl") - def validate_intl(cls, v): + @field_validator("name_intl", "short_description_intl", "description_intl") + @classmethod + def validate_intl(cls, v: Dict[str, str]): # None is also a valid type if v is None: return v @@ -193,7 +195,7 @@ def create_oonirun_link( ) db_oonirun_link.nettests.append( models.OONIRunLinkNettest( - **nettest.dict(), + **nettest.model_dump(), date_created=now, nettest_index=nettest_index, revision=revision, @@ -443,7 +445,7 @@ def get_oonirun_link_engine_descriptor( revision_number: Annotated[ str, Path( - regex="^(latest|\\d+)$", + pattern="^(latest|\\d+)$", error_messages={ "regex": "invalid revision number specified, must be 'latest' or a number" }, @@ -491,7 +493,7 @@ def get_oonirun_link_revision( revision_number: Annotated[ str, Path( - regex="^(latest|\\d+)$", + pattern="^(latest|\\d+)$", error_messages={ "regex": "invalid revision number specified, must be 'latest' or a number" },