feat: oonifindings service

ooniapi/common/src/common/auth.py

@@ -0,0 +1,54 @@
+import hashlib
+from typing import Optional, Dict, Any 
+import jwt
+
+def hash_email_address(email_address: str, key: str) -> str:
+    em = email_address.encode()
+    return hashlib.blake2b(em, key=key.encode("utf-8"), digest_size=16).hexdigest()
+
+
+def decode_jwt(token: str, key: str, **kw) -> Dict[str, Any]:
+    tok = jwt.decode(token, key, algorithms=["HS256"], **kw)
+    return tok
+
+
+def create_jwt(payload: dict, key: str) -> str:
+    token = jwt.encode(payload, key, algorithm="HS256")
+    if isinstance(token, bytes):
+        return token.decode()
+    else:
+        return token
+
+
+def get_client_token(authorization: str, jwt_encryption_key: str):
+    try:
+        assert authorization.startswith("Bearer ")
+        token = authorization[7:]
+        return decode_jwt(token, audience="user_auth", key=jwt_encryption_key)
+    except:
+        return None
+
+
+def get_client_role(authorization: str, jwt_encryption_key: str) -> str:
+    """Raise exception for unlogged users"""
+    tok = get_client_token(authorization, jwt_encryption_key)
+    assert tok
+    return tok["role"]
+
+
+def get_account_id_or_none(
+    authorization: str, jwt_encryption_key: str
+) -> Optional[str]:
+    """Returns None for unlogged users"""
+    tok = get_client_token(authorization, jwt_encryption_key)
+    if tok:
+        return tok["account_id"]
+    return None
+
+
+def get_account_id_or_raise(authorization: str, jwt_encryption_key: str) -> str:
+    """Raise exception for unlogged users"""
+    tok = get_client_token(authorization, jwt_encryption_key)
+    if tok:
+        return tok["account_id"]
+    raise Exception

ooniapi/common/src/common/clickhouse_utils.py

@@ -61,7 +61,7 @@ def optimize_table(db: clickhouse_driver.Client, tblname: str) -> None:
 
 def raw_query(
     db: clickhouse_driver.Client, query: Query, query_params: dict, query_prio=1
-):
+) -> int:
     settings = {"priority": query_prio, "max_execution_time": 300}
     q = db.execute(query, query_params, with_column_types=True, settings=settings)
     return q

ooniapi/common/src/common/config.py

@@ -16,6 +16,7 @@ class Settings(BaseSettings):
     jwt_encryption_key: str = "CHANGEME"
     prometheus_metrics_password: str = "CHANGEME"
     account_id_hashing_key: str = "CHANGEME"
+    collector_id: str = "CHANGEME"
     session_expiry_days: int = 10
     login_expiry_days: int = 10
 

ooniapi/common/src/common/dependencies.py

@@ -3,7 +3,7 @@
 
 from fastapi import Depends
 from fastapi import HTTPException, Header
-from .utils import get_client_token
+from .auth import get_client_token
 from .config import Settings
 
 

ooniapi/common/src/common/exceptions.py

@@ -0,0 +1,25 @@
+from typing import Dict
+
+from fastapi import HTTPException
+
+
+class BaseOONIException(HTTPException):
+    def __init__(self, detail: str, headers: Dict | None):
+        super.__init__(status_code=400, detail=detail, headers=headers)
+
+
+class InvalidRequest(BaseOONIException):
+    def __init__(self):
+        super().__init__(detail="invalid parameters in the request")
+
+
+class OwnershipPermissionError(BaseOONIException):
+    def __init__(self):
+        super().__init__(
+            detail = "attempted to create, update or delete an item belonging to another user"
+        )
+
+
+class DatabaseQueryError(BaseOONIException):
+    def __init__(self):
+        super().__init__(detail="")

ooniapi/common/src/common/utils.py

@@ -1,11 +1,12 @@
 from csv import DictWriter
 from io import StringIO
+from sys import byteorder
+from os import urandom
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import List
+from fastapi import Response
 from fastapi.responses import JSONResponse
 
-import jwt
-
 
 log = logging.getLogger(__name__)
 
@@ -31,6 +32,15 @@ def jerror(msg, code=400, **kw) -> JSONResponse:
     return JSONResponse(content=dict(msg=msg, **kw), status_code=code, headers=headers)
 
 
+def setcacheresponse(interval: str, response: Response):
+    max_age = int(interval[:-1]) * INTERVAL_UNITS[interval[-1]]
+    response.headers["Cache-Control"] = f"max-age={max_age}"
+
+
+def setnocacheresponse(response: Response):
+    response.headers["Cache-Control"] = "no-cache"
+
+
 def commasplit(p: str) -> List[str]:
     assert p is not None
     out = set(p.split(","))
@@ -60,57 +70,10 @@ def convert_to_csv(r) -> str:
     return result
 
 
-def decode_jwt(token: str, key: str, **kw) -> Dict[str, Any]:
-    tok = jwt.decode(token, key, algorithms=["HS256"], **kw)
-    return tok
-
-
-def create_jwt(payload: dict, key: str) -> str:
-    token = jwt.encode(payload, key, algorithm="HS256")
-    if isinstance(token, bytes):
-        return token.decode()
-    else:
-        return token
-
-
-def get_client_token(authorization: str, jwt_encryption_key: str):
+def generate_random_intuid(collector_id: str) -> int:
     try:
-        assert authorization.startswith("Bearer ")
-        token = authorization[7:]
-        return decode_jwt(token, audience="user_auth", key=jwt_encryption_key)
-    except:
-        return None
-
-
-def get_client_role(authorization: str, jwt_encryption_key: str) -> str:
-    """Raise exception for unlogged users"""
-    tok = get_client_token(authorization, jwt_encryption_key)
-    assert tok
-    return tok["role"]
-
-
-def get_account_id_or_none(
-    authorization: str, jwt_encryption_key: str
-) -> Optional[str]:
-    """Returns None for unlogged users"""
-    tok = get_client_token(authorization, jwt_encryption_key)
-    if tok:
-        return tok["account_id"]
-    return None
-
-
-def get_account_id_or_raise(authorization: str, jwt_encryption_key: str) -> str:
-    """Raise exception for unlogged users"""
-    tok = get_client_token(authorization, jwt_encryption_key)
-    if tok:
-        return tok["account_id"]
-    raise Exception
-
-
-def get_account_id(authorization: str, jwt_encryption_key: str):
-    # TODO: switch to get_account_id_or_none
-    tok = get_client_token(authorization, jwt_encryption_key)
-    if not tok:
-        return jerror("Authentication required", 401)
-
-    return tok["account_id"]
+        collector_id = int(collector_id)
+    except ValueError:
+        collector_id = 0
+    randint = int.from_bytes(urandom(4), byteorder)
+    return randint * 100 + collector_id

ooniapi/services/oonifindings/.dockerignore

@@ -0,0 +1,10 @@
+.DS_Store
+*.log
+*.pyc
+*.swp
+*.env
+.coverage
+coverage.xml
+dist/
+.venv/
+__pycache__/

ooniapi/services/oonifindings/.gitignore

@@ -0,0 +1,3 @@
+/dist
+/coverage_html
+*.coverage*

ooniapi/services/oonifindings/Dockerfile

@@ -0,0 +1,33 @@
+# Python builder
+FROM python:3.11-bookworm as builder
+ARG BUILD_LABEL=docker
+
+WORKDIR /build
+
+RUN python -m pip install hatch
+
+COPY . /build
+
+# When you build stuff on macOS you end up with ._ files
+# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them
+RUN find /build -type f -name '._*' -delete
+
+RUN echo "$BUILD_LABEL" > /build/src/oonifindings/BUILD_LABEL
+
+RUN hatch build
+
+### Actual image running on the host
+FROM python:3.11-bookworm as runner
+
+WORKDIR /app
+
+COPY --from=builder /build/README.md /app/
+COPY --from=builder /build/dist/*.whl /app/
+RUN pip install /app/*whl && rm /app/*whl
+
+COPY --from=builder /build/alembic/ /app/alembic/
+COPY --from=builder /build/alembic.ini /app/
+RUN rm -rf /app/alembic/__pycache__
+
+CMD ["uvicorn", "oonifindings.main:app", "--host", "0.0.0.0", "--port", "80"]
+EXPOSE 80

ooniapi/services/oonifindings/LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ooniapi/services/oonifindings/README.md

@@ -0,0 +1,21 @@
+# oonifindings
+
+[![PyPI - Version](https://img.shields.io/pypi/v/oonifindings.svg)](https://pypi.org/project/oonifindings)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/oonifindings.svg)](https://pypi.org/project/oonifindings)
+
+-----
+
+**Table of Contents**
+
+- [Installation](#installation)
+- [License](#license)
+
+## Installation
+
+```console
+pip install oonifindings
+```
+
+## License
+
+`oonifindings` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license.

ooniapi/services/oonifindings/migrations/clickhouse_init_tables.sql

@@ -0,0 +1,28 @@
+-- Create tables for integration tests
+
+CREATE TABLE default.incidents 
+(
+    `id` String,
+    `title` String,
+    `short_description` String,
+    `text` String,
+    `start_time` Datetime DEFAULT now(),
+    `end_time` Nullable(Datetime),
+    `create_time` Datetime,
+    `update_time` Datetime DEFAULT now(),
+    `creator_account_id` FixedString(32),
+    `reported_by` String,
+    `email_address` String,
+    `event_type` LowCardinality(String),
+    `published` UInt8,
+    `deleted` UInt8 DEFAULT 0,
+    `CCs` Array(FixedString(2)),
+    `ASNs` Array(String),
+    `domains` Array(String),
+    `tags` Array(String),
+    `links` Array(String),
+    `test_names` Array(String), 
+)
+ENGINE=ReplacingMergeTree
+ORDER BY (create_time, creator_account_id, id)
+SETTINGS index_granularity = 8192;