Merge branch 'main' into pydantic2

* main:
  🍼 Ensure pip (#195)
  Adds pip to venv (#192)
  Adds venv to start of PATH (#191)
  Activates venv in production image (#190)
  🎬 Row actions (#188)
  🤖 Update dependencies (#186)
  📩 MMIF Download (#182)
  ⛓️ Metaflow links (#185)
  🎱 Fields: `Finished`, `Successful` (#184)
  ➕ Add MMIF to batch (#181)
  🤖 Update dependencies (#177)
  🔏 Requires valid access tokens for API calls (#173)
  🏞 `MMIF` source  (#168)
  ⌛ Expires all (#172)
  🤖 Update dependencies (#171)
  📄 Page title (#162)
  Fixes MediaFilesGuidsField, combines usage (#170)

.dockerignore

@@ -1 +1,2 @@
-.pdm.toml
+.pdm.toml
+__pycache__

.env.sample

@@ -0,0 +1,13 @@
+# Postgres URL for Chowda DB.
+# Format: postgresql://{user}:{password}@{host}:{port}/{database}
+DB_URL=''
+
+# Env vars beginning with "AUTH0_" are used for authentication and authorization with Auth0.
+# These values can be obtained by logging into our Auth0 account.
+AUTH0_DOMAIN=''
+AUTH0_CLIENT_ID=''
+AUTH0_CLIENT_SECRET=''
+AUTH0_API_AUDIENCE=''
+
+# Secret value for securing session data using SessionMiddleware
+CHOWDA_SECRET=''

.gitguardian.yaml

@@ -0,0 +1,5 @@
+secret:
+  ignored-matches:
+  - match: 73f13c96260784cee71e44a63798a1a6716f19d6581a2ce28b391c4d8449bd19
+    name: Default jwt.io Bearer Token, for auth tests - tests/routers/test_events.py
+version: 2

.github/workflows/CI.yml

@@ -2,9 +2,6 @@ name: 🧪 Integration Tests
 
 on: [push, pull_request, workflow_dispatch]
 
-env:
-  DB_URL: postgresql://postgres:postgres@postgres:5432/chowda-test
-
 jobs:
   tests:
     name: ⚗️ Application Tests
@@ -13,7 +10,7 @@ jobs:
     with:
       pdm_args: -G test,ci
       pytest_args: -n auto --nbmake -ra -s
-      pg_db: chowda-test
+      pg_db: chowda
 
   lint:
     name: 👕 Lint

.gitignore

@@ -23,4 +23,9 @@ ci.toml
 
 # Metaflow generated files
 metaflow.s3.*
-.metaflow
+.metaflow
+
+.cache_ggshield
+
+.env.*
+!.env.sample

.vscode/launch.json

@@ -18,20 +18,6 @@
       "jinja": false,
       "justMyCode": false
     },
-    {
-      "name": "Chowda: tests",
-      "type": "python",
-      "request": "launch",
-      "cwd": "${workspaceFolder}",
-      "module": "pytest",
-      "args": [
-        "-v",
-        "-n",
-        "auto"
-      ],
-      "jinja": false,
-      "justMyCode": false
-    },
     {
       "name": "Metaflow: Run",
       "type": "python",

Dockerfile

@@ -1,11 +1,11 @@
 ###########################
 # 'base' build stage, common to all build stages
 ###########################
-FROM python:3.11 as base
+FROM python:3.11-slim as base
 
 # Set working dir to /app, where all Chowda code lives.
 WORKDIR /app
-RUN pip install -U pip
+RUN pip install -U pip pdm
 
 # Copy app code to container
 COPY pyproject.toml pdm.lock README.md ./
@@ -20,14 +20,12 @@ COPY migrations migrations
 # 'dev' build stage
 ###########################
 FROM base as dev
-# Install PDM dependency manager
-RUN pip install pdm
 # Configure pdm to instal dependencies into ./__pypyackages__/
 RUN pdm config python.use_venv false
 # Configure python to use pep582 with local __pypyackages__
 ENV PYTHONPATH=/usr/local/lib/python3.11/site-packages/pdm/pep582
 # Add local packages to $PATH
-ENV PATH=$PATH:/app/__pypackages__/3.11/bin/
+ENV PATH=/app/__pypackages__/3.11/bin/:$PATH
 
 # Install dev dependencies with pdm
 RUN pdm install -G dev
@@ -57,13 +55,34 @@ CMD poetry run locust
 
 
 ###########################
-# 'production' build stage
+# 'base' build stage for production
 ############################
-FROM base as production
-RUN pip install .[production]
+FROM base as build
+RUN apt update && apt install -y gcc libpq-dev git
 
-COPY static static
+RUN pdm config venv.with_pip True
+RUN pdm install -G production
+
+# Install pip into the virtual environment
+RUN /app/.venv/bin/python -m ensurepip
+
+###########################
+# 'production' final production image
+############################
+FROM python:3.11-slim as production
+WORKDIR /app
+
+RUN apt update && apt install -y libpq-dev
+RUN apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=build /app/ /app/
 COPY templates templates
+COPY static static
+
 ENV CHOWDA_ENV=production
+ENV PATH="/app/.venv/bin:$PATH"
+
+EXPOSE 8000
 
 CMD gunicorn chowda.app:app -b 0.0.0.0:8000 -w 2 --worker-class uvicorn.workers.UvicornWorker

chowda/app.py

@@ -2,7 +2,7 @@
 
 Main Chowda application"""
 
-from fastapi import FastAPI
+from fastapi import Depends, FastAPI
 from fastapi.staticfiles import StaticFiles
 from starlette.middleware import Middleware
 from starlette.middleware.sessions import SessionMiddleware
@@ -13,9 +13,11 @@
 from chowda.admin import Admin
 from chowda.api import api
 from chowda.auth import OAuthProvider
+from chowda.auth.utils import get_admin_user, verified_access_token
 from chowda.config import SECRET, STATIC_DIR, TEMPLATES_DIR
 from chowda.db import engine
 from chowda.models import (
+    MMIF,
     Batch,
     ClamsApp,
     Collection,
@@ -33,6 +35,7 @@
     DashboardView,
     MediaFileView,
     MetaflowRunView,
+    MMIFView,
     PipelineView,
     SonyCiAssetView,
     UserView,
@@ -51,8 +54,10 @@
 )
 app.mount('/static', StaticFiles(directory=STATIC_DIR), name='static')
 
-app.include_router(api, prefix='/api')
-app.include_router(dashboard, prefix='/dashboard')
+app.include_router(api, prefix='/api', dependencies=[Depends(verified_access_token)])
+app.include_router(
+    dashboard, prefix='/dashboard', dependencies=[Depends(get_admin_user)]
+)
 
 
 # Create admin
@@ -70,11 +75,12 @@
 admin.add_view(MediaFileView(MediaFile, icon='fa fa-file-video'))
 admin.add_view(SonyCiAssetView(SonyCiAsset, icon='fa fa-file-video'))
 admin.add_view(CollectionView(Collection, icon='fa fa-folder'))
-admin.add_view(BatchView(Batch, icon='fa fa-folder', label='Batches'))
+admin.add_view(BatchView(Batch, icon='fa fa-folder'))
 admin.add_view(ClamsAppView(ClamsApp, icon='fa fa-box'))
 admin.add_view(PipelineView(Pipeline, icon='fa fa-boxes-stacked'))
 admin.add_view(UserView(User, icon='fa fa-users'))
 admin.add_view(MetaflowRunView(MetaflowRun, icon='fa fa-person-running'))
+admin.add_view(MMIFView(MMIF, icon='fa fa-person-running'))
 
 
 # Mount admin to app

chowda/auth/oauth.py

@@ -9,7 +9,12 @@
 from starlette_admin import BaseAdmin
 from starlette_admin.auth import AdminUser, AuthMiddleware, AuthProvider
 
-from chowda.config import AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET, AUTH0_DOMAIN
+from chowda.config import (
+    AUTH0_API_AUDIENCE,
+    AUTH0_CLIENT_ID,
+    AUTH0_CLIENT_SECRET,
+    AUTH0_DOMAIN,
+)
 
 oauth = OAuth()
 oauth.register(
@@ -20,6 +25,7 @@
         'scope': 'openid profile email',
     },
     server_metadata_url=f'https://{AUTH0_DOMAIN}/.well-known/openid-configuration',
+    authorize_params={'audience': AUTH0_API_AUDIENCE},
 )
 
 

chowda/auth/utils.py

@@ -1,17 +1,35 @@
-from typing import Annotated
+from typing import Annotated, List, Set
 
 from fastapi import Depends, HTTPException, Request, status
 from pydantic import BaseModel, Field
 
-from chowda.config import API_AUDIENCE
+from chowda.config import AUTH0_API_AUDIENCE, AUTH0_JWKS_URL
+
+unauthorized_redirect = HTTPException(
+    status_code=status.HTTP_303_SEE_OTHER,
+    detail='Not Authorized',
+    headers={'Location': '/admin'},
+)
+
+unauthorized = HTTPException(
+    status_code=status.HTTP_401_UNAUTHORIZED,
+    detail='Not Authorized',
+)
 
 
-class UserToken(BaseModel):
+class OAuthAccessToken(BaseModel):
+    """OAuth Authorization token"""
+
+    sub: str
+    permissions: List[str] = []
+
+
+class OAuthUser(BaseModel):
     """ID Token model for authorization."""
 
     name: str
     email: str | None = None
-    roles: set[str] = Field(set(), alias=f'{API_AUDIENCE}/roles')
+    roles: set[str] = Field(set(), alias=f'{AUTH0_API_AUDIENCE}/roles')
 
     @property
     def is_admin(self) -> bool:
@@ -24,29 +42,107 @@ def is_clammer(self) -> bool:
         return 'clammer' in self.roles
 
 
-unauthorized = HTTPException(
-    status_code=status.HTTP_303_SEE_OTHER,
-    detail='Not Authorized',
-    headers={'Location': '/admin'},
-)
-
-
-def get_user(request: Request) -> UserToken:
+def get_oauth_user(request: Request) -> OAuthUser:
     """Get the user token from the session."""
     user = request.session.get('user', None)
     if not user:
         request.session['error'] = 'Not Logged In'
-        raise unauthorized
-    return UserToken(**user)
+        raise unauthorized_redirect
+    return OAuthUser(**user)
 
 
-def admin_user(
-    request: Request, user: Annotated[UserToken, Depends(get_user)]
-) -> UserToken:
+def get_admin_user(
+    request: Request, user: Annotated[OAuthUser, Depends(get_oauth_user)]
+) -> OAuthUser:
     """Check if the user has the admin role using FastAPI Depends.
     If not, sets a session error and raises an HTTPException."""
     if not user.is_admin:
         request.session['error'] = 'Not Authorized'
-        raise unauthorized
+        raise unauthorized_redirect
 
     return user
+
+
+def unverified_access_token(request: Request) -> str:
+    """Extract and return the unverified access token from the Authorization header.
+    Raises an HTTPUnauthorizedException if the header is missing or malformed."""
+    auth_header = request.headers.get('Authorization', None)
+
+    if not auth_header:
+        raise HTTPException(status_code=401, detail="Missing Authorization header")
+    if not auth_header.startswith('Bearer '):
+        raise HTTPException(
+            status_code=401,
+            detail="Bearer token malformed or missing in Authorization header",
+        )
+
+    # Return the access token from the Authorization header,
+    # i.e. the string without the "Bearer " prefix
+    return auth_header.replace('Bearer ', '')
+
+
+def jwt_signing_key(
+    unverified_access_token: Annotated[str, Depends(unverified_access_token)]
+) -> str:
+    """Get the JWT signing key from the JWKS URL."""
+    from jwt import PyJWKClient
+
+    jwks_client = PyJWKClient(AUTH0_JWKS_URL)
+    try:
+        signing_key = jwks_client.get_signing_key_from_jwt(unverified_access_token)
+        return signing_key.key
+
+    except Exception as exc:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED, detail=str(exc)
+        ) from exc
+
+
+def verified_access_token(
+    request: Request,
+    unverified_access_token: Annotated[str, Depends(unverified_access_token)],
+    jwt_signing_key: Annotated[str, Depends(jwt_signing_key)],
+) -> OAuthAccessToken:
+    """Decodes and verifies an access token. If any exceptions occur, an
+    HTTPUnauthorizedException is raised from the original exception."""
+    from jwt import decode
+
+    try:
+        decoded_token = decode(
+            unverified_access_token,
+            jwt_signing_key,
+            algorithms=['RS256', 'HS256'],
+            audience='https://chowda.wgbh-mla.org/api',
+        )
+        return OAuthAccessToken(**decoded_token)
+
+    except Exception as exc:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED, detail=str(exc)
+        ) from exc
+
+
+def permissions(permissions: str | List[str] | Set[str]) -> None:
+    """Dependency function to check if token has required permissions.
+
+    Args:
+        permissions (str, List, Set): Required permissions. Can be a str, list, or set.
+    Examples:
+        @app.get('/users/', dependencies=[Depends(permissions('read:users'))])
+    """
+    if isinstance(permissions, (str, list)):
+        permissions: set = {permissions}
+
+    def _permissions(
+        token: Annotated[OAuthAccessToken, Depends(verified_access_token)],
+    ) -> None:
+        """Verify token has all required permissions, or raise a 403 Forbidden exception
+        with the missing permissions in the detail message."""
+        missing_permissions = permissions - set(token.permissions)
+        if missing_permissions:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=f'Missing required permissions: {missing_permissions}',
+            )
+
+    return _permissions