feat: oonifindings service (#850)

* feat(oonifindings): findings ooniapi service

Create oonifindings service with clickhouse support

* refactor: switch to using pytest-docker

* feat: add build files

* feat: oonifindings service v1

* integrate changes from common into oonirun

* migrate ooniauth to new common

* code cleanup

* add github action for oonifindings tests

* introduce api tests for oonifindings

* add tests for oonifindings router

* extend router tests for oonifindings

* refactor: remove null account and use client

* refactor: use cliet without token

* delete unrequired fixtures

* test for cache headers in response

.github/workflows/test_ooniapi_oonifindings.yml

@@ -0,0 +1,25 @@
+name: test ooniapi/oonifindings
+on: push
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Run all tests
+        run: make test-cov
+        working-directory: ./ooniapi/services/oonifindings/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: oonifindings
+          working-directory: ./ooniapi/services/oonifindings/

ooniapi/common/src/common/auth.py

@@ -0,0 +1,70 @@
+import hashlib
+from typing import Optional, Dict, Any 
+import jwt
+
+
+def hash_email_address(email_address: str, key: str) -> str:
+    em = email_address.encode()
+    return hashlib.blake2b(em, key=key.encode("utf-8"), digest_size=16).hexdigest()
+
+
+def check_email_address(
+    authorization: str,
+    jwt_encryption_key: str,
+    email_address: str,
+    key: str
+) -> bool:
+    account_id = get_account_id_or_raise(authorization, jwt_encryption_key=jwt_encryption_key)
+    hashed = hash_email_address(email_address, key=key)
+    if account_id == hashed:
+        return True
+    return False
+
+
+def decode_jwt(token: str, key: str, **kw) -> Dict[str, Any]:
+    tok = jwt.decode(token, key, algorithms=["HS256"], **kw)
+    return tok
+
+
+def create_jwt(payload: dict, key: str) -> str:
+    token = jwt.encode(payload, key, algorithm="HS256")
+    if isinstance(token, bytes):
+        return token.decode()
+    else:
+        return token
+
+
+def get_client_token(authorization: str, jwt_encryption_key: str):
+    try:
+        assert authorization.startswith("Bearer ")
+        token = authorization[7:]
+        return decode_jwt(token, audience="user_auth", key=jwt_encryption_key)
+    except:
+        return None
+
+
+def get_client_role(authorization: str, jwt_encryption_key: str) -> str:
+    """Raise exception for unlogged users"""
+    tok = get_client_token(authorization, jwt_encryption_key)
+    try:
+        assert tok
+        return tok["role"]
+    except:
+        return None
+
+def get_account_id_or_none(
+    authorization: str, jwt_encryption_key: str
+) -> Optional[str]:
+    """Returns None for unlogged users"""
+    tok = get_client_token(authorization, jwt_encryption_key)
+    if tok:
+        return tok["account_id"]
+    return None
+
+
+def get_account_id_or_raise(authorization: str, jwt_encryption_key: str) -> str:
+    """Raise exception for unlogged users"""
+    tok = get_client_token(authorization, jwt_encryption_key)
+    if tok:
+        return tok["account_id"]
+    raise Exception

ooniapi/common/src/common/clickhouse_utils.py

@@ -61,7 +61,7 @@ def optimize_table(db: clickhouse_driver.Client, tblname: str) -> None:
 
 def raw_query(
     db: clickhouse_driver.Client, query: Query, query_params: dict, query_prio=1
-):
+) -> int:
     settings = {"priority": query_prio, "max_execution_time": 300}
     q = db.execute(query, query_params, with_column_types=True, settings=settings)
     return q

ooniapi/common/src/common/config.py

@@ -16,6 +16,7 @@ class Settings(BaseSettings):
     jwt_encryption_key: str = "CHANGEME"
     prometheus_metrics_password: str = "CHANGEME"
     account_id_hashing_key: str = "CHANGEME"
+    collector_id: str = "CHANGEME"
     session_expiry_days: int = 10
     login_expiry_days: int = 10
 

ooniapi/common/src/common/dependencies.py

@@ -3,7 +3,7 @@
 
 from fastapi import Depends
 from fastapi import HTTPException, Header
-from .utils import get_client_token
+from .auth import get_client_token
 from .config import Settings
 
 
@@ -29,6 +29,9 @@ async def verify_jwt(
             tok = get_client_token(authorization, settings.jwt_encryption_key)
         except:
             raise HTTPException(detail="Authentication required", status_code=401)
+
+        if not tok:
+            raise HTTPException(detail="Authentication required", status_code=401)
         if tok["role"] not in roles:
             raise HTTPException(detail="Role not authorized", status_code=401)
 

ooniapi/common/src/common/utils.py

@@ -1,11 +1,12 @@
 from csv import DictWriter
 from io import StringIO
+from sys import byteorder
+from os import urandom
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import List
+from fastapi import Response
 from fastapi.responses import JSONResponse
 
-import jwt
-
 
 log = logging.getLogger(__name__)
 
@@ -31,6 +32,15 @@ def jerror(msg, code=400, **kw) -> JSONResponse:
     return JSONResponse(content=dict(msg=msg, **kw), status_code=code, headers=headers)
 
 
+def setcacheresponse(interval: str, response: Response):
+    max_age = int(interval[:-1]) * INTERVAL_UNITS[interval[-1]]
+    response.headers["Cache-Control"] = f"max-age={max_age}"
+
+
+def setnocacheresponse(response: Response):
+    response.headers["Cache-Control"] = "no-cache"
+
+
 def commasplit(p: str) -> List[str]:
     assert p is not None
     out = set(p.split(","))
@@ -60,44 +70,10 @@ def convert_to_csv(r) -> str:
     return result
 
 
-def decode_jwt(token: str, key: str, **kw) -> Dict[str, Any]:
-    tok = jwt.decode(token, key, algorithms=["HS256"], **kw)
-    return tok
-
-
-def create_jwt(payload: dict, key: str) -> str:
-    token = jwt.encode(payload, key, algorithm="HS256")
-    if isinstance(token, bytes):
-        return token.decode()
-    else:
-        return token
-
-
-def get_client_token(authorization: str, jwt_encryption_key: str) -> Dict[str, Any]:
-    assert authorization.startswith("Bearer ")
-    token = authorization[7:]
-    return decode_jwt(token, audience="user_auth", key=jwt_encryption_key)
-
-
-def get_client_role(authorization: str, jwt_encryption_key: str) -> str:
-    """Raise exception for unlogged users"""
-    tok = get_client_token(authorization, jwt_encryption_key)
-    assert tok
-    return tok["role"]
-
-
-def get_account_id_or_none(
-    authorization: str, jwt_encryption_key: str
-) -> Optional[str]:
-    """Returns None for unlogged users"""
+def generate_random_intuid(collector_id: str) -> int:
     try:
-        tok = get_client_token(authorization, jwt_encryption_key)
-        return tok["account_id"]
-    except:
-        return None
-
-
-def get_account_id_or_raise(authorization: str, jwt_encryption_key: str) -> str:
-    """Raise exception for unlogged users"""
-    tok = get_client_token(authorization, jwt_encryption_key)
-    return tok["account_id"]
+        collector_id = int(collector_id)
+    except ValueError:
+        collector_id = 0
+    randint = int.from_bytes(urandom(4), byteorder)
+    return randint * 100 + collector_id

ooniapi/services/ooniauth/src/ooniauth/routers/v1.py

@@ -6,7 +6,8 @@
 import jwt
 
 from fastapi import APIRouter, Depends, Query, HTTPException, Header, Path
-from pydantic import Field, validator
+from pydantic import Field
+from pydantic.functional_validators import field_validator
 from pydantic import EmailStr
 from typing_extensions import Annotated
 
@@ -22,7 +23,7 @@
 from ..common.dependencies import get_settings, role_required
 from ..common.config import Settings
 from ..common.routers import BaseModel
-from ..common.utils import (
+from ..common.auth import (
     create_jwt,
     decode_jwt,
     get_client_token,
@@ -42,7 +43,7 @@ class UserRegister(BaseModel):
     )
     redirect_to: str = Field(title="redirect to this URL")
 
-    @validator("redirect_to")
+    @field_validator("redirect_to")
     def validate_redirect_to(cls, v):
         u = urlparse(v)
         if u.scheme != "https":

ooniapi/services/ooniauth/src/ooniauth/routers/v2.py

@@ -6,7 +6,8 @@
 import jwt
 
 from fastapi import APIRouter, Depends, HTTPException, Header
-from pydantic import Field, validator
+from pydantic import Field
+from pydantic.functional_validators import field_validator
 from pydantic import EmailStr
 from typing_extensions import Annotated
 
@@ -23,7 +24,7 @@
 from ..common.dependencies import get_settings
 from ..common.config import Settings
 from ..common.routers import BaseModel
-from ..common.utils import (
+from ..common.auth import (
     create_jwt,
     decode_jwt,
     get_client_token,
@@ -43,7 +44,7 @@ class CreateUserLogin(BaseModel):
     )
     redirect_to: str = Field(title="redirect to this URL")
 
-    @validator("redirect_to")
+    @field_validator("redirect_to")
     def validate_redirect_to(cls, v):
         u = urlparse(v)
         if u.scheme != "https":

ooniapi/services/ooniauth/src/ooniauth/utils.py

@@ -6,7 +6,7 @@
 
 import sqlalchemy as sa
 
-from .common.utils import create_jwt
+from .common.auth import create_jwt
 
 VALID_REDIRECT_TO_FQDN = (
     "explorer.ooni.org",

ooniapi/services/ooniauth/tests/test_auth_v1.py

@@ -1,6 +1,4 @@
 from urllib.parse import parse_qs, urlparse
-from ooniauth.common.utils import decode_jwt
-from ooniauth.main import app
 from freezegun import freeze_time
 
 from html.parser import HTMLParser

ooniapi/services/ooniauth/tests/test_auth_v2.py

@@ -1,6 +1,4 @@
 from urllib.parse import parse_qs, urlparse
-from ooniauth.common.utils import decode_jwt
-from ooniauth.main import app
 from freezegun import freeze_time
 
 

ooniapi/services/oonifindings/.dockerignore

@@ -0,0 +1,10 @@
+.DS_Store
+*.log
+*.pyc
+*.swp
+*.env
+.coverage
+coverage.xml
+dist/
+.venv/
+__pycache__/

ooniapi/services/oonifindings/.gitignore

@@ -0,0 +1,3 @@
+/dist
+/coverage_html
+*.coverage*

ooniapi/services/oonifindings/Dockerfile

@@ -0,0 +1,29 @@
+# Python builder
+FROM python:3.11-bookworm as builder
+ARG BUILD_LABEL=docker
+
+WORKDIR /build
+
+RUN python -m pip install hatch
+
+COPY . /build
+
+# When you build stuff on macOS you end up with ._ files
+# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them
+RUN find /build -type f -name '._*' -delete
+
+RUN echo "$BUILD_LABEL" > /build/src/oonifindings/BUILD_LABEL
+
+RUN hatch build
+
+### Actual image running on the host
+FROM python:3.11-bookworm as runner
+
+WORKDIR /app
+
+COPY --from=builder /build/README.md /app/
+COPY --from=builder /build/dist/*.whl /app/
+RUN pip install /app/*whl && rm /app/*whl
+
+CMD ["uvicorn", "oonifindings.main:app", "--host", "0.0.0.0", "--port", "80"]
+EXPOSE 80

ooniapi/services/oonifindings/LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.