feat(oonifindings): add postgresql support (#862)

* feat(oonifindings): add postgresql data models and migrations

* feat(oonifindings): replace clickhouse with postgresql (sqlalchemy)

* feat(oonifindings): replace service tests to use postgresql

* fix: skip failing integration test

* refactor: changes to model from code review

* refactor: change incident_id to finding_id in table

* refactor: assemble multiple queries in single

* feat: add finding_slug column

* feat(oonifindings): add TODO for finding_slug

* fix: remove changes from oonirun database test

ooniapi/common/src/common/alembic/versions/981d92cf8790_init_tables.py

@@ -84,4 +84,4 @@ def upgrade() -> None:
 
 def downgrade() -> None:
     op.drop_table("oonirun_nettest")
-    op.drop_table("oonirun")
+    op.drop_table("oonirun")

ooniapi/common/src/common/alembic/versions/a037e908f3a0_init_oonifindings.py

@@ -0,0 +1,54 @@
+"""init oonifindings
+
+Revision ID: a037e908f3a0
+Revises: c9119c05cf42
+Create Date: 2024-07-17 16:45:25.752551
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'a037e908f3a0'
+down_revision: Union[str, None] = "c9119c05cf42"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "oonifinding",
+        sa.Column(
+            "finding_id",
+            sa.String(),
+            nullable=False,
+            primary_key=True,
+        ),
+        sa.Column("finding_slug", sa.String(), nullable=True),
+        sa.Column("create_time", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("update_time", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("start_time", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("end_time", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("creator_account_id", sa.String(), nullable=False),
+        sa.Column("title", sa.String(), nullable=False),
+        sa.Column("short_description", sa.String(), nullable=False),
+        sa.Column("text", sa.String(), nullable=False),
+        sa.Column("reported_by", sa.String(), nullable=False),
+        sa.Column("email_address", sa.String(), nullable=False),
+        sa.Column("event_type", sa.String(), nullable=False),
+        sa.Column("published", sa.Integer(), nullable=False),
+        sa.Column("deleted", sa.Integer(), nullable=False, default=0),
+        sa.Column("country_codes", sa.JSON(), nullable=True),
+        sa.Column("asns", sa.JSON(), nullable=True),
+        sa.Column("domains", sa.JSON(), nullable=True),
+        sa.Column("tags", sa.JSON(), nullable=True),
+        sa.Column("links", sa.JSON(), nullable=True),
+        sa.Column("test_names", sa.JSON(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("oonifinding")

ooniapi/common/src/common/models.py

@@ -1,5 +1,4 @@
 from datetime import datetime, timezone
-from typing import List, Dict, Any
 from sqlalchemy.types import DateTime, TypeDecorator
 
 

ooniapi/services/oonifindings/pyproject.toml

@@ -16,7 +16,9 @@ dependencies = [
   "statsd ~= 4.0.1",
   "uvicorn ~= 0.25.0",
   "httpx ~= 0.26.0",
+  "psycopg2 ~= 2.9.9",
   "pyjwt ~= 2.8.0",
+  "alembic ~= 1.13.1",
   "prometheus-fastapi-instrumentator ~= 6.1.0",
   "prometheus-client",
 ]
@@ -55,17 +57,14 @@ include = ["BUILD_LABEL"]
 packages = ["src/oonifindings"]
 artifacts = ["BUILD_LABEL"]
 
-[tool.hatch.metadata]
-allow-direct-references = true
-
 [tool.hatch.envs.default]
 dependencies = [
   "pytest",
   "pytest-cov",
   "click",
   "black",
   "pytest-asyncio",
-  "pytest-docker",
+  "pytest-postgresql",
 ]
 path = ".venv/"
 
@@ -78,12 +77,8 @@ cov = ["test-cov", "cov-report"]
 [[tool.hatch.envs.all.matrix]]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
-[tool.hatch.envs.types]
-dependencies = [
-  "mypy>=1.0.0",
-]
-[tool.hatch.envs.types.scripts]
-check = "mypy --install-types --non-interactive {args:src/oonifindings tests}"
+[tool.pytest.ini_options]
+addopts = ["--import-mode=importlib"]
 
 [tool.coverage.run]
 source_pkgs = ["oonifindings", "tests"]

ooniapi/services/oonifindings/src/oonifindings/dependencies.py

@@ -2,14 +2,19 @@
 
 from fastapi import Depends
 
-from clickhouse_driver import Client as Clickhouse
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
 
 from .common.config import Settings
 from .common.dependencies import get_settings
 
-def get_clickhouse_session(settings: Annotated[Settings, Depends(get_settings)]):
-    db = Clickhouse.from_url(settings.clickhouse_url)
+
+def get_postgresql_session(settings: Annotated[Settings, Depends(get_settings)]):
+    engine = create_engine(settings.postgresql_url)
+    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+    db = SessionLocal()
     try:
         yield db
-    finally: 
-        db.disconnect()
+    finally:
+        db.close()

ooniapi/services/oonifindings/src/oonifindings/main.py

@@ -8,14 +8,13 @@
 
 from prometheus_fastapi_instrumentator import Instrumentator
 
+from . import models
 from .routers import v1
 
-from .dependencies import get_settings, get_clickhouse_session
+from .dependencies import get_settings, get_postgresql_session
 from .common.version import get_build_label, get_pkg_version
-from .common.clickhouse_utils import query_click
 from .common.metrics import mount_metrics
 
-
 pkg_name = "oonifindings"
 
 pkg_version = get_pkg_version(pkg_name)
@@ -67,17 +66,12 @@ class HealthStatus(BaseModel):
 @app.get("/health")
 async def health(
     settings=Depends(get_settings),
-    db=Depends(get_clickhouse_session),
+    db=Depends(get_postgresql_session),
 ):
     errors = []
 
     try:
-        query = f"""SELECT id, update_time, start_time, end_time, reported_by,
-        title, event_type, published, CCs, ASNs, domains, tags, test_names,
-        links, short_description, email_address, create_time, creator_account_id 
-        FROM incidents FINAL
-        """
-        query_click(db=db, query=query, query_params={})
+        db.query(models.OONIFinding).limit(1).all()
     except Exception as exc:
         log.error(exc)
         errors.append("db_error")
@@ -86,17 +80,17 @@ async def health(
         err = "bad_jwt_secret"
         log.error(err)
         errors.append(err)
-    
+
     if settings.prometheus_metrics_password == "CHANGEME":
         err = "bad_prometheus_password"
         log.error(err)
         errors.append(err)
 
     if len(errors) > 0:
         raise HTTPException(status_code=400, detail="health check failed")
-    
+
     status = "ok"
-    
+
     return {
         "status": status,
         "errors": errors,

ooniapi/services/oonifindings/src/oonifindings/models.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+from typing import List
+from sqlalchemy import String
+from sqlalchemy.orm import Mapped
+from sqlalchemy.orm import mapped_column
+
+from .common.models import UtcDateTime
+from .common.postgresql import Base
+
+
+class OONIFinding(Base):
+    __tablename__ = "oonifinding"
+
+    finding_id: Mapped[str] = mapped_column(String, primary_key=True)
+    # TODO(decfox): this is nullable for now. We should
+    # make this a non-nullable field eventually and have an endpoint
+    # where we can query findings using the finding_slug.
+    finding_slug: Mapped[str] = mapped_column(String, nullable=True)
+
+    create_time: Mapped[datetime] = mapped_column(UtcDateTime())
+    update_time: Mapped[datetime] = mapped_column(UtcDateTime())
+    start_time: Mapped[datetime] = mapped_column(UtcDateTime(), nullable=True)
+    end_time: Mapped[datetime] = mapped_column(UtcDateTime(), nullable=True)
+    creator_account_id: Mapped[str] = mapped_column(String(32))
+
+    title: Mapped[str] = mapped_column()
+    short_description: Mapped[str] = mapped_column()
+    text: Mapped[str] = mapped_column()
+    reported_by: Mapped[str] = mapped_column()
+    email_address: Mapped[str] = mapped_column()
+    event_type: Mapped[str] = mapped_column()
+    published: Mapped[int] = mapped_column()
+    deleted: Mapped[int] = mapped_column(default=0)
+
+    country_codes: Mapped[List[str]] = mapped_column(nullable=True)
+    asns: Mapped[List[str]] = mapped_column(nullable=True)
+    domains: Mapped[List[str]] = mapped_column(nullable=True)
+    tags: Mapped[List[str]] = mapped_column(nullable=True)
+    links: Mapped[List[str]] = mapped_column(nullable=True)
+    test_names: Mapped[List[str]] = mapped_column(nullable=True)