Merge branch 'devel' for release

.github/dependabot.yml

@@ -0,0 +1,17 @@
+# Referenced from:
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+      time: "00:00"
+      timezone: "Europe/London"
+    open-pull-requests-limit: 5
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/run-tests.yml

@@ -28,6 +28,16 @@ jobs:
           MYSQL_PASSWORD: "test"
           MYSQL_DATABASE: "study_notify"
 
+      porch:
+        image: "ghcr.io/wtsi-npg/python-3.10-npg-porch-2.0.0"
+        ports:
+          - "8081:8081"
+        options: >-
+          --health-cmd "curl -f http://localhost:8081"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
     steps:
       - uses: actions/checkout@v4
 
@@ -52,4 +62,3 @@ jobs:
       - name: Run linter (ruff)
         run: |
           poetry run ruff check --output-format=github .
-

Dockerfile.dev

@@ -0,0 +1,61 @@
+FROM python:3.12-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-utils \
+    ca-certificates \
+    git \
+    locales \
+    unattended-upgrades && \
+    unattended-upgrade -v && \
+    locale-gen en_GB en_GB.UTF-8 && \
+    localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8 && \
+    apt-get remove -q -y unattended-upgrades && \
+    apt-get autoremove -q -y && \
+    apt-get clean -q -y && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8 \
+    TZ=/Etc/UTC
+
+WORKDIR /app
+
+ARG APP_USER=appuser
+ARG APP_UID=1000
+ARG APP_GID=$APP_UID
+
+RUN groupadd --gid $APP_GID $APP_USER && \
+    useradd --uid $APP_UID --gid $APP_GID --shell /bin/bash --create-home $APP_USER
+
+ARG POETRY_VERSION="1.8.3"
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_CACHE_DIR=/app/.poetry
+
+RUN python -m venv /app/venv && \
+    . /app/venv/bin/activate && \
+    pip install --no-cache-dir "poetry==$POETRY_VERSION"
+
+COPY pyproject.toml poetry.lock /app/
+
+RUN . /app/venv/bin/activate && \
+    poetry install --no-root
+
+COPY . /app
+
+RUN . /app/venv/bin/activate && \
+    poetry install && \
+    rm -rf "$POETRY_CACHE_DIR"
+
+RUN chown -R $APP_USER:$APP_USER /app
+
+USER $APP_USER
+
+ENTRYPOINT ["/app/docker/entrypoint.sh"]
+
+CMD ["/bin/bash", "-c", "sleep infinity"]

README.md

@@ -20,10 +20,12 @@ is sent to `porch`.
 
 ## Scope
 
-The current version implements notifications for PacBio sequencing platform
+The current version implements notifications for PacBio and ONT sequencing platform
 customers.
 
-## Running the scripts
+## PacBio
+
+### Running the scripts
 
 To register recently QC-ed entities as tasks with `porch`
 
@@ -41,3 +43,80 @@ Processing includes claiming one task, sending per-study emails and updating the
 status of the `porch` task to `DONE`.
 
 The test data directory has an example of a [configuration file](tests/data/qc_state_app_config.ini).
+
+### ONT
+
+#### Configuration
+
+Configuration requires an INI-format file with the following sections:
+
+```ini
+[MySQL MLWH]
+dbhost = <MySQL host>
+dbport = <MySQL port>
+dbuser = <MySQL user>
+dbpassword = <MySQL password>
+dbschema = <MySQL schema>
+
+[PORCH]
+url = <Porch server http URL>
+admin_token = <Porch server admin authentication token>
+pipeline_token = <Porch server pipeline authentication token>
+
+[MAIL]
+domain = <email FQDN>
+```
+
+The first time a new pipeline is created on a particular `porch` service, the following two
+steps are requiredL
+
+- Register the pipeline with `porch`:
+
+```bash
+ npg_ont_event_notification --verbose --colour --conf-file path/to/config.ini register
+```
+
+This creates a new record of the pipeline and its version in the `porch` database. This
+step requires an admin token for the `porch` service to be in the configuration file. The
+`--verbose` and `--colour` flags are optional, but help to see the progress of the script.
+
+- Obtain a new pipelne authentication token:
+
+```bash
+ npg_ont_event_notification --verbose --colour --conf-file path/to/config.ini token
+```
+
+This creates a new token and prints it to STDOUT. Once the token is obtained, it should be
+added to the configuration file. The token cannot be retrieved again. This token is used to
+authenticate with the `porch` service when submitting and running tasks.
+
+#### Running the script
+
+There are two parts to the notification process:
+
+1. Finding new ONT runs of interest and adding them to the notification pipeline as tasks.
+2. Processing the tasks and sending the notifications.
+
+Each of these steps is typically run as a separate cron job. Adding the same task multiple
+times is safe, as `porch` will process is only once.
+
+- Find new ONT runs and add tasks:
+
+```bash
+<run disocvery script> | npg_ont_event_notification --verbose --colour --conf-file path/to/config.ini --log-config path/to/logging.ini add
+```
+
+The discovery script can be anything that writes [baton](http://wtsi-npg.github.io/baton)
+format JSON to STDOUT, one JSON object per line e.g. the `locate-data-objects` script from
+https://github.com/wtsi-npg/npg-irods-python . The JSON must include the metadata associated
+with the path it represents.
+
+- Process tasks:
+
+```bash
+npg_ont_event_notification --conf-file path/to/config.ini --log-config path/to/logging.ini run
+```
+
+This will claim up to 100 tasks from the `porch` service, process them and mark them as done
+if they are successful, or retry them if they fail. The claim size is fixed and should be
+sufficient to clear all tasks each time the script is run.

docker-compose.yml

@@ -0,0 +1,38 @@
+services:
+  mysql-server:
+    image: mysql:8.0
+    restart: always
+    ports:
+      - "127.0.0.1:3306:3306"
+    environment:
+      MYSQL_USER: "test"
+      MYSQL_PASSWORD: "test"
+      MYSQL_DATABASE: "study_notify"
+      MYSQL_RANDOM_ROOT_PASSWORD: "true"
+    healthcheck:
+      test: mysqladmin ping
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  porch-server:
+    image: "ghcr.io/wtsi-npg/python-3.10-npg-porch-2.0.0"
+    restart: always
+    ports:
+      - "127.0.0.1:8081:8081"
+    healthcheck:
+        test: curl -f http://localhost:8081
+        interval: 10s
+        timeout: 5s
+        retries: 10
+
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    restart: always
+    depends_on:
+      mysql-server:
+        condition: service_healthy
+      porch-server:
+        condition: service_healthy

docker/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This virtualenv is created by the Dockerfile
+source /app/venv/bin/activate
+
+exec "$@"