From 196c52e73217e74d2a18a0d3b8817c5f34c3f8ba Mon Sep 17 00:00:00 2001 From: Keith James Date: Fri, 8 Nov 2024 13:50:26 +0000 Subject: [PATCH] Add documentation Add to README docstrings. Update the test config file so that it matches the Docker Compose setup for ONT. --- README.md | 83 ++++++++++++++++++++++++++++- src/npg_notify/ont/event.py | 17 +++--- tests/data/ont_event_app_config.ini | 10 ++-- tests/ont/test_porch.py | 4 +- 4 files changed, 96 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index fafd1b7..bf59544 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,12 @@ is sent to `porch`. ## Scope -The current version implements notifications for PacBio sequencing platform +The current version implements notifications for PacBio and ONT sequencing platform customers. -## Running the scripts +## PacBio + +### Running the scripts To register recently QC-ed entities as tasks with `porch` @@ -41,3 +43,80 @@ Processing includes claiming one task, sending per-study emails and updating the status of the `porch` task to `DONE`. The test data directory has an example of a [configuration file](tests/data/qc_state_app_config.ini). + +### ONT + +#### Configuration + +Configuration requires an INI-format file with the following sections: + +```ini +[MySQL MLWH] +dbhost = +dbport = +dbuser = +dbpassword = +dbschema = + +[PORCH] +url = +admin_token = +pipeline_token = + +[MAIL] +domain = +``` + +The first time a new pipeline is created on a particular `porch` service, the following two +steps are requiredL + +- Register the pipeline with `porch`: + +```bash + npg_ont_event_notification --verbose --colour --conf-file path/to/config.ini register +``` + +This creates a new record of the pipeline and its version in the `porch` database. This +step requires an admin token for the `porch` service to be in the configuration file. The +`--verbose` and `--colour` flags are optional, but help to see the progress of the script. + +- Obtain a new pipelne authentication token: + +```bash + npg_ont_event_notification --verbose --colour --conf-file path/to/config.ini token +``` + +This creates a new token and prints it to STDOUT. Once the token is obtained, it should be +added to the configuration file. The token cannot be retrieved again. This token is used to +authenticate with the `porch` service when submitting and running tasks. + +#### Running the script + +There are two parts to the notification process: + +1. Finding new ONT runs of interest and adding them to the notification pipeline as tasks. +2. Processing the tasks and sending the notifications. + +Each of these steps is typically run as a separate cron job. Adding the same task multiple +times is safe, as `porch` will process is only once. + +- Find new ONT runs and add tasks: + +```bash + | npg_ont_event_notification --verbose --colour --conf-file path/to/config.ini --log-config path/to/logging.ini add +``` + +The discovery script can be anything that writes [baton](http://wtsi-npg.github.io/baton) +format JSON to STDOUT, one JSON object per line e.g. the `locate-data-objects` script from +https://github.com/wtsi-npg/npg-irods-python . The JSON must include the metadata associated +with the path it represents. + +- Process tasks: + +```bash +npg_ont_event_notification --conf-file path/to/config.ini --log-config path/to/logging.ini run +``` + +This will claim up to 100 tasks from the `porch` service, process them and mark them as done +if they are successful, or retry them if they fail. The claim size is fixed and should be +sufficient to clear all tasks each time the script is run. diff --git a/src/npg_notify/ont/event.py b/src/npg_notify/ont/event.py index c70fa78..f246279 100644 --- a/src/npg_notify/ont/event.py +++ b/src/npg_notify/ont/event.py @@ -70,16 +70,17 @@ authentication token, and the MySQL MLWH database connection details. The configuration file should be in INI format and have the following sections: + [MySQL MLWH] + dbhost = + dbport = + dbuser = + dbpassword = + dbschema = + [PORCH] url = - token = - - [MySQL MLWH] - host = - port = - user = - password = - db = + admin_token = + pipeline_token = [MAIL] domain = diff --git a/tests/data/ont_event_app_config.ini b/tests/data/ont_event_app_config.ini index 917d8a7..cd2ab60 100644 --- a/tests/data/ont_event_app_config.ini +++ b/tests/data/ont_event_app_config.ini @@ -1,18 +1,16 @@ -# Configuration for the npg_notify package. - -[MLWH] +[MySQL MLWH] dbuser = test dbpassword = test -dbhost = 127.0.0.1 +dbhost = mysql-server dbport = 3306 dbschema = study_notify [PORCH] -url = http://127.0.0.1:8081 +url = http://porch-server:8081 admin_token = 00000000000000000000000000000000 [MAIL] -domain = some.com +domain = not-a-real-domain.sanger.ac.uk diff --git a/tests/ont/test_porch.py b/tests/ont/test_porch.py index c389ff7..458e7db 100644 --- a/tests/ont/test_porch.py +++ b/tests/ont/test_porch.py @@ -80,7 +80,7 @@ def test_configure(self): assert config is not None assert config.admin_token == "0" * 32 assert config.pipeline_token is None - assert config.url == "http://127.0.0.1:8081" + assert config.url == "http://porch-server:8081" @m.context("When configured from both an INI file and environment variables") @m.it("Loads the configuration and falls back to environment variables") @@ -98,7 +98,7 @@ def test_configure_env(self): assert config is not None assert config.admin_token == "0" * 32 assert config.pipeline_token == pipeline_token - assert config.url == "http://127.0.0.1:8081" + assert config.url == "http://porch-server:8081" @m.context("When a pipeline has been defined") @m.it("Can be registered")