Add email notification for ONT run events

Add dependency on Partisan (for baton JSON IO - no requirement for
iRODS access or baton itself).

Add version() function to report the notification pipeline version for
ONT.

Add necessary ONT tables to MLWH ORM.

Add dev Dockerfile and compose file.

Rename some QC event-specific resources to have "QC" in the name, to
allow ONT event-specific resources to be added.

.github/workflows/run-tests.yml

@@ -28,6 +28,16 @@ jobs:
           MYSQL_PASSWORD: "test"
           MYSQL_DATABASE: "study_notify"
 
+      porch:
+        image: "ghcr.io/wtsi-npg/python-3.10-npg-porch-2.0.0"
+        ports:
+          - "8081:8081"
+        options: >-
+          --health-cmd "curl -f http://localhost:8081"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
     steps:
       - uses: actions/checkout@v4
 
@@ -52,4 +62,3 @@ jobs:
       - name: Run linter (ruff)
         run: |
           poetry run ruff check --output-format=github .
-

Dockerfile.dev

@@ -0,0 +1,61 @@
+FROM python:3.12-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-utils \
+    ca-certificates \
+    git \
+    locales \
+    unattended-upgrades && \
+    unattended-upgrade -v && \
+    locale-gen en_GB en_GB.UTF-8 && \
+    localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8 && \
+    apt-get remove -q -y unattended-upgrades && \
+    apt-get autoremove -q -y && \
+    apt-get clean -q -y && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8 \
+    TZ=/Etc/UTC
+
+WORKDIR /app
+
+ARG APP_USER=appuser
+ARG APP_UID=1000
+ARG APP_GID=$APP_UID
+
+RUN groupadd --gid $APP_GID $APP_USER && \
+    useradd --uid $APP_UID --gid $APP_GID --shell /bin/bash --create-home $APP_USER
+
+ARG POETRY_VERSION="1.8.3"
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_CACHE_DIR=/app/.poetry
+
+RUN python -m venv /app/venv && \
+    . /app/venv/bin/activate && \
+    pip install --no-cache-dir "poetry==$POETRY_VERSION"
+
+COPY pyproject.toml poetry.lock /app/
+
+RUN . /app/venv/bin/activate && \
+    poetry install --no-root
+
+COPY . /app
+
+RUN . /app/venv/bin/activate && \
+    poetry install && \
+    rm -rf "$POETRY_CACHE_DIR"
+
+RUN chown -R $APP_USER:$APP_USER /app
+
+USER $APP_USER
+
+ENTRYPOINT ["/app/docker/entrypoint.sh"]
+
+CMD ["/bin/bash", "-c", "sleep infinity"]

docker-compose.yml

@@ -0,0 +1,38 @@
+services:
+  mysql-server:
+    image: mysql:8.0
+    restart: always
+    ports:
+      - "127.0.0.1:3306:3306"
+    environment:
+      MYSQL_USER: "test"
+      MYSQL_PASSWORD: "test"
+      MYSQL_DATABASE: "study_notify"
+      MYSQL_RANDOM_ROOT_PASSWORD: "true"
+    healthcheck:
+      test: mysqladmin ping
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  porch-server:
+    image: wsinpg/python-3.10-npg-porch-2.0.0:latest
+    restart: always
+    ports:
+      - "127.0.0.1:8081:8081"
+    healthcheck:
+        test: curl -f http://localhost:8081
+        interval: 10s
+        timeout: 5s
+        retries: 10
+
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    restart: always
+    depends_on:
+      mysql-server:
+        condition: service_healthy
+      porch-server:
+        condition: service_healthy

docker/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This virtualenv is created by the Dockerfile
+source /app/venv/bin/activate
+
+exec "$@"

pyproject.toml

@@ -1,13 +1,14 @@
 [tool.poetry]
 name = "npg_notify"
-version = "0.0.1"
 description = "Utility for client notifications"
+version = "0.0.0"
 authors = ["Marina Gourtovaia"]
 license = "GPL-3.0-or-later"
 readme = "README.md"
 
 [tool.poetry.scripts]
-npg_qc_state_notification = "npg_notify.porch_wrapper.qc_state:run" 
+npg_qc_state_notification = "npg_notify.porch_wrapper.qc_state:run"
+npg_ont_event_notification = "npg_notify.ont.event:main"
 
 [tool.poetry.dependencies]
 python = "^3.11"
@@ -16,15 +17,26 @@ SQLAlchemy-Utils = "^0.41.2"
 cryptography = "^41.0.3"
 PyYAML = "^6.0.0"
 npg_porch_cli = { git="https://github.com/wtsi-npg/npg_porch_cli.git", tag="0.1.0" }
+partisan = { url = "https://github.com/wtsi-npg/partisan/releases/download/2.13.0/partisan-2.13.0.tar.gz" }
+npg-python-lib = { url = "https://github.com/wtsi-npg/npg-python-lib/releases/download/0.3.2/npg_python_lib-0.3.2.tar.gz" }
+requests = "^2.32.0"
+structlog = "^24.4.0"
+black = "^24.10.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^8.2.2"
+pytest-it = "^0.1.5"
 requests-mock = "^1.12.1"
 ruff = "^0.4.9"
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
+build-backend = "poetry_dynamic_versioning.backend"
+
+[tool.poetry-dynamic-versioning]
+enable = true
+vcs = "git"
+pattern = "default-unprefixed"
 
 [tool.ruff]
 # Set the maximum line length to 79.

pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+pythonpath = src tests
+testpaths = tests
+python_functions = test_*
+log_cli = False
+log_cli_level = ERROR

src/npg_notify/__init__.py

@@ -0,0 +1,25 @@
+#
+# Copyright © 2024 Genome Research Ltd. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import importlib.metadata
+
+__version__ = importlib.metadata.version("npg_notify")
+
+
+def version() -> str:
+    """Return the current version."""
+    return __version__

src/npg_notify/data/resources/ont_event_email_template.txt

@@ -0,0 +1,9 @@
+The ONT run for experiment $experiment_name, flowcell $flowcell_id has been $event.
+The data are available in iRODS at the following path:
+
+$path
+
+This is an automated email from NPG. You are receiving it because you are registered
+as a contact for one or more of the Studies listed below:
+
+$studies

src/npg_notify/db/mlwh.py

@@ -4,7 +4,7 @@
 #   Marina Gourtovaia <[email protected]>
 #   Kieron Taylor <[email protected]>
 #
-# This file is part of npg_notifications software package..
+# This file is part of npg_notifications software package.
 #
 # npg_notifications is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -45,6 +45,39 @@ class Base(DeclarativeBase):
     pass
 
 
+class Sample(Base):
+    __tablename__ = "sample"
+
+    id_sample_tmp = mapped_column(Integer, primary_key=True, autoincrement=True)
+    id_lims = mapped_column(String(10), nullable=False)
+    id_sample_lims = mapped_column(String(20), nullable=False)
+    created = mapped_column(DateTime, nullable=False)
+    last_updated = mapped_column(DateTime, nullable=False)
+    recorded_at = mapped_column(DateTime, nullable=False)
+    consent_withdrawn = mapped_column(Integer, nullable=False, default=0)
+    name = mapped_column(String(255), index=True)
+    organism = mapped_column(String(255))
+    accession_number = mapped_column(String(50), index=True)
+    common_name = mapped_column(String(255))
+    cohort = mapped_column(String(255))
+    sanger_sample_id = mapped_column(String(255), index=True)
+    supplier_name = mapped_column(String(255), index=True)
+    public_name = mapped_column(String(255))
+    donor_id = mapped_column(String(255))
+    date_of_consent_withdrawn = mapped_column(DateTime)
+    marked_as_consent_withdrawn_by = mapped_column(String(255))
+
+    oseq_flowcell: Mapped["OseqFlowcell"] = relationship(
+        "OseqFlowcell", back_populates="sample"
+    )
+
+    def __repr__(self):
+        return (
+            f"<Sample pk={self.id_sample_tmp} id_sample_lims={self.id_sample_lims} "
+            f"name='{self.name}'>"
+        )
+
+
 class Study(Base):
     """A representation for the 'study' table."""
 
@@ -63,6 +96,9 @@ class Study(Base):
         ),
     )
 
+    oseq_flowcell: Mapped["OseqFlowcell"] = relationship(
+        "OseqFlowcell", back_populates="study"
+    )
     study_users: Mapped[set["StudyUser"]] = relationship()
 
     def __repr__(self):
@@ -116,6 +152,47 @@ class StudyNotFoundError(Exception):
     pass
 
 
+class OseqFlowcell(Base):
+    __tablename__ = "oseq_flowcell"
+
+    id_oseq_flowcell_tmp = mapped_column(Integer, primary_key=True, autoincrement=True)
+    id_flowcell_lims = mapped_column(String(255), nullable=False)
+    last_updated = mapped_column(DateTime, nullable=False)
+    recorded_at = mapped_column(DateTime, nullable=False)
+    id_sample_tmp = mapped_column(
+        ForeignKey("sample.id_sample_tmp"), nullable=False, index=True
+    )
+    id_study_tmp = mapped_column(
+        ForeignKey("study.id_study_tmp"), nullable=False, index=True
+    )
+    experiment_name = mapped_column(String(255), nullable=False)
+    instrument_name = mapped_column(String(255), nullable=False)
+    instrument_slot = mapped_column(Integer, nullable=False)
+    id_lims = mapped_column(String(10), nullable=False)
+    pipeline_id_lims = mapped_column(String(255))
+    requested_data_type = mapped_column(String(255))
+    tag_identifier = mapped_column(String(255))
+    tag_sequence = mapped_column(String(255))
+    tag_set_id_lims = mapped_column(String(255))
+    tag_set_name = mapped_column(String(255))
+    tag2_identifier = mapped_column(String(255))
+    tag2_sequence = mapped_column(String(255))
+    tag2_set_id_lims = mapped_column(String(255))
+    tag2_set_name = mapped_column(String(255))
+    flowcell_id = mapped_column(String(255))
+    run_id = mapped_column(String(255))
+
+    sample: Mapped["Sample"] = relationship("Sample", back_populates="oseq_flowcell")
+    study: Mapped["Study"] = relationship("Study", back_populates="oseq_flowcell")
+
+    def __repr__(self):
+        return (
+            f"<OseqFlowcell expt_name={self.experiment_name} "
+            f"slot={self.instrument_slot} "
+            f"flowcell={self.flowcell_id}>"
+        )
+
+
 def get_study_contacts(session: Session, study_id: str) -> list[str]:
     """Retrieves emails of study contacts from the mlwh database.