Merge pull request #9 from kjsanger/feature/ont-event

Add email notification for ONT run events

.github/workflows/run-tests.yml

@@ -28,6 +28,16 @@ jobs:
           MYSQL_PASSWORD: "test"
           MYSQL_DATABASE: "study_notify"
 
+      porch:
+        image: "ghcr.io/wtsi-npg/python-3.10-npg-porch-2.0.0"
+        ports:
+          - "8081:8081"
+        options: >-
+          --health-cmd "curl -f http://localhost:8081"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
     steps:
       - uses: actions/checkout@v4
 
@@ -52,4 +62,3 @@ jobs:
       - name: Run linter (ruff)
         run: |
           poetry run ruff check --output-format=github .
-

Dockerfile.dev

@@ -0,0 +1,61 @@
+FROM python:3.12-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-utils \
+    ca-certificates \
+    git \
+    locales \
+    unattended-upgrades && \
+    unattended-upgrade -v && \
+    locale-gen en_GB en_GB.UTF-8 && \
+    localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8 && \
+    apt-get remove -q -y unattended-upgrades && \
+    apt-get autoremove -q -y && \
+    apt-get clean -q -y && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8 \
+    TZ=/Etc/UTC
+
+WORKDIR /app
+
+ARG APP_USER=appuser
+ARG APP_UID=1000
+ARG APP_GID=$APP_UID
+
+RUN groupadd --gid $APP_GID $APP_USER && \
+    useradd --uid $APP_UID --gid $APP_GID --shell /bin/bash --create-home $APP_USER
+
+ARG POETRY_VERSION="1.8.3"
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_CACHE_DIR=/app/.poetry
+
+RUN python -m venv /app/venv && \
+    . /app/venv/bin/activate && \
+    pip install --no-cache-dir "poetry==$POETRY_VERSION"
+
+COPY pyproject.toml poetry.lock /app/
+
+RUN . /app/venv/bin/activate && \
+    poetry install --no-root
+
+COPY . /app
+
+RUN . /app/venv/bin/activate && \
+    poetry install && \
+    rm -rf "$POETRY_CACHE_DIR"
+
+RUN chown -R $APP_USER:$APP_USER /app
+
+USER $APP_USER
+
+ENTRYPOINT ["/app/docker/entrypoint.sh"]
+
+CMD ["/bin/bash", "-c", "sleep infinity"]

docker-compose.yml

@@ -0,0 +1,38 @@
+services:
+  mysql-server:
+    image: mysql:8.0
+    restart: always
+    ports:
+      - "127.0.0.1:3306:3306"
+    environment:
+      MYSQL_USER: "test"
+      MYSQL_PASSWORD: "test"
+      MYSQL_DATABASE: "study_notify"
+      MYSQL_RANDOM_ROOT_PASSWORD: "true"
+    healthcheck:
+      test: mysqladmin ping
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  porch-server:
+    image: "ghcr.io/wtsi-npg/python-3.10-npg-porch-2.0.0"
+    restart: always
+    ports:
+      - "127.0.0.1:8081:8081"
+    healthcheck:
+        test: curl -f http://localhost:8081
+        interval: 10s
+        timeout: 5s
+        retries: 10
+
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    restart: always
+    depends_on:
+      mysql-server:
+        condition: service_healthy
+      porch-server:
+        condition: service_healthy

docker/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This virtualenv is created by the Dockerfile
+source /app/venv/bin/activate
+
+exec "$@"

pyproject.toml

@@ -1,13 +1,14 @@
 [tool.poetry]
 name = "npg_notify"
-version = "0.0.1"
 description = "Utility for client notifications"
+version = "0.0.0"
 authors = ["Marina Gourtovaia"]
 license = "GPL-3.0-or-later"
 readme = "README.md"
 
 [tool.poetry.scripts]
-npg_qc_state_notification = "npg_notify.porch_wrapper.qc_state:run" 
+npg_qc_state_notification = "npg_notify.porch_wrapper.qc_state:run"
+npg_ont_event_notification = "npg_notify.ont.event:main"
 
 [tool.poetry.dependencies]
 python = "^3.11"
@@ -16,15 +17,25 @@ SQLAlchemy-Utils = "^0.41.2"
 cryptography = "^41.0.3"
 PyYAML = "^6.0.0"
 npg_porch_cli = { git="https://github.com/wtsi-npg/npg_porch_cli.git", tag="0.1.0" }
+partisan = { url = "https://github.com/wtsi-npg/partisan/releases/download/2.13.0/partisan-2.13.0.tar.gz" }
+npg-python-lib = { url = "https://github.com/wtsi-npg/npg-python-lib/releases/download/0.3.2/npg_python_lib-0.3.2.tar.gz" }
+requests = "^2.32.0"
+structlog = "^24.4.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^8.2.2"
+pytest-it = "^0.1.5"
 requests-mock = "^1.12.1"
 ruff = "^0.4.9"
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
+build-backend = "poetry_dynamic_versioning.backend"
+
+[tool.poetry-dynamic-versioning]
+enable = true
+vcs = "git"
+pattern = "default-unprefixed"
 
 [tool.ruff]
 # Set the maximum line length to 79.

pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+pythonpath = src tests
+testpaths = tests
+python_functions = test_*
+log_cli = False
+log_cli_level = ERROR

src/npg_notify/__init__.py

@@ -0,0 +1,25 @@
+#
+# Copyright © 2024 Genome Research Ltd. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import importlib.metadata
+
+__version__ = importlib.metadata.version("npg_notify")
+
+
+def version() -> str:
+    """Return the current version."""
+    return __version__

src/npg_notify/data/resources/ont_event_email_template.txt

@@ -0,0 +1,9 @@
+The ONT run for experiment $experiment_name, flowcell $flowcell_id has been $event.
+The data are available in iRODS at the following path:
+
+$path
+
+This is an automated email from NPG. You are receiving it because you are registered
+as a contact for one or more of the Studies listed below:
+
+$studies

src/npg_notify/db/mlwh.py

@@ -4,7 +4,7 @@
 #   Marina Gourtovaia <[email protected]>
 #   Kieron Taylor <[email protected]>
 #
-# This file is part of npg_notifications software package..
+# This file is part of npg_notifications software package.
 #
 # npg_notifications is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -45,6 +45,39 @@ class Base(DeclarativeBase):
     pass
 
 
+class Sample(Base):
+    __tablename__ = "sample"
+
+    id_sample_tmp = mapped_column(Integer, primary_key=True, autoincrement=True)
+    id_lims = mapped_column(String(10), nullable=False)
+    id_sample_lims = mapped_column(String(20), nullable=False)
+    created = mapped_column(DateTime, nullable=False)
+    last_updated = mapped_column(DateTime, nullable=False)
+    recorded_at = mapped_column(DateTime, nullable=False)
+    consent_withdrawn = mapped_column(Integer, nullable=False, default=0)
+    name = mapped_column(String(255), index=True)
+    organism = mapped_column(String(255))
+    accession_number = mapped_column(String(50), index=True)
+    common_name = mapped_column(String(255))
+    cohort = mapped_column(String(255))
+    sanger_sample_id = mapped_column(String(255), index=True)
+    supplier_name = mapped_column(String(255), index=True)
+    public_name = mapped_column(String(255))
+    donor_id = mapped_column(String(255))
+    date_of_consent_withdrawn = mapped_column(DateTime)
+    marked_as_consent_withdrawn_by = mapped_column(String(255))
+
+    oseq_flowcell: Mapped["OseqFlowcell"] = relationship(
+        "OseqFlowcell", back_populates="sample"
+    )
+
+    def __repr__(self):
+        return (
+            f"<Sample pk={self.id_sample_tmp} id_sample_lims={self.id_sample_lims} "
+            f"name='{self.name}'>"
+        )
+
+
 class Study(Base):
     """A representation for the 'study' table."""
 
@@ -63,6 +96,9 @@ class Study(Base):
         ),
     )
 
+    oseq_flowcell: Mapped["OseqFlowcell"] = relationship(
+        "OseqFlowcell", back_populates="study"
+    )
     study_users: Mapped[set["StudyUser"]] = relationship()
 
     def __repr__(self):
@@ -116,6 +152,47 @@ class StudyNotFoundError(Exception):
     pass
 
 
+class OseqFlowcell(Base):
+    __tablename__ = "oseq_flowcell"
+
+    id_oseq_flowcell_tmp = mapped_column(Integer, primary_key=True, autoincrement=True)
+    id_flowcell_lims = mapped_column(String(255), nullable=False)
+    last_updated = mapped_column(DateTime, nullable=False)
+    recorded_at = mapped_column(DateTime, nullable=False)
+    id_sample_tmp = mapped_column(
+        ForeignKey("sample.id_sample_tmp"), nullable=False, index=True
+    )
+    id_study_tmp = mapped_column(
+        ForeignKey("study.id_study_tmp"), nullable=False, index=True
+    )
+    experiment_name = mapped_column(String(255), nullable=False)
+    instrument_name = mapped_column(String(255), nullable=False)
+    instrument_slot = mapped_column(Integer, nullable=False)
+    id_lims = mapped_column(String(10), nullable=False)
+    pipeline_id_lims = mapped_column(String(255))
+    requested_data_type = mapped_column(String(255))
+    tag_identifier = mapped_column(String(255))
+    tag_sequence = mapped_column(String(255))
+    tag_set_id_lims = mapped_column(String(255))
+    tag_set_name = mapped_column(String(255))
+    tag2_identifier = mapped_column(String(255))
+    tag2_sequence = mapped_column(String(255))
+    tag2_set_id_lims = mapped_column(String(255))
+    tag2_set_name = mapped_column(String(255))
+    flowcell_id = mapped_column(String(255))
+    run_id = mapped_column(String(255))
+
+    sample: Mapped["Sample"] = relationship("Sample", back_populates="oseq_flowcell")
+    study: Mapped["Study"] = relationship("Study", back_populates="oseq_flowcell")
+
+    def __repr__(self):
+        return (
+            f"<OseqFlowcell expt_name={self.experiment_name} "
+            f"slot={self.instrument_slot} "
+            f"flowcell={self.flowcell_id}>"
+        )
+
+
 def get_study_contacts(session: Session, study_id: str) -> list[str]:
     """Retrieves emails of study contacts from the mlwh database.