Merge pull request #74 from wtsi-npg/devel

Release 2.0

.github/dependabot.yml

@@ -0,0 +1,17 @@
+version: 2
+updates:
+- package-ecosystem: "pip"
+  directory: "/"
+  schedule:
+    interval: "daily"
+    time: "00:00"
+    timezone: "Europe/London"
+  open-pull-requests-limit: 5
+
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "daily"
+    time: "00:00"
+    timezone: "Europe/London"
+  open-pull-requests-limit: 5

.github/workflows/python-app.yml

@@ -15,9 +15,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.10
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: "3.10"
     - name: Install dependencies

CHANGELOG.md

@@ -0,0 +1,38 @@
+# Change Log for npg_porch Project
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/).
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+## [Unreleased]
+
+## [2.0.0] - 2024-07-31
+
+### Added
+
+* A note about installing on macOS
+* An endpoint for creating pipeline tokens
+* A development Docker file
+* A Token model
+
+### Changed
+
+* Moved the project to the standard Python project layout.
+* Reorganised and renamed the modules:
+    1. `npg.porch` namespace is collapsed into `npg_porch`,
+    2. `npg.porchdb` is reorganised into `npg_porch.db`,
+    3. the entry point of the application is relocated and
+    renamed `npg/porch_server.py` -> `npg_porch/server.py`
+* Moved the test configuration file to the `tests` directory.
+* Changed the HTTP code for the `pipeline` endpoint from 409 to 422 in case of
+  bad input.
+* A Token model is returned rather than a bare string by the endpoint which
+creates tokens for a pipeline.
+* A request to create an already existing task is no longer invalid, does not
+raise an error.
+* Updated the Task model so that it is impossible to create or update the task
+  without explicitly specifying the new task status.
+* pysqlite3 is moved from the production to test dependencies.
+
+## [1.0.0] - 2023-10-16
+
+Initial release

README.md

@@ -36,23 +36,33 @@ To run the server, please execute the following from the root directory:
 ```bash
 bash
 pip3 install -e .
-cd server
+cd src
 mkdir -p logs
 export DB_URL=postgresql+asyncpg://npg_rw:$PASS@npg_porch_db:$PORT/$DATABASE
 export DB_SCHEMA='non_default'
-uvicorn npg.main:app --host 0.0.0.0 --port 8080 --reload --log-config logging.json
+uvicorn npg_porch.server:app --host 0.0.0.0 --port 8080 --reload --log-config logging.json
 ```
 
 and open your browser at `http://localhost:8080` to see links to the docs.
 
+On macOS you will need to ensure that a version of the `sqlite3` library that supports SQLite extensions
+is used when installing the `pysqlite3` package. The system library on macOS does not, so an alternative
+such as the one provided by MacPorts or Homebrew should be used. For example, when using MacPorts this
+can be done by setting the `CPPFLAGS` environment variable before running the `pip install` command:
+
+```
+export CPPFLAGS="-I/opt/local/include"
+```
+
+
 The server will not start without `DB_URL` in the environment
 
 ## Running in production
 
 When you want HTTPS, logging and all that jazz:
 
 ```bash
-uvicorn main:app --workers 2 --host 0.0.0.0 --port 8080 --log-config ~/logging.json --ssl-keyfile ~/.ssh/key.pem --ssl-certfile ~/.ssh/cert.pem --ssl-ca-certs /usr/local/share/ca-certificates/institute_ca.crt
+uvicorn server:app --workers 2 --host 0.0.0.0 --port 8080 --log-config ~/logging.json --ssl-keyfile ~/.ssh/key.pem --ssl-certfile ~/.ssh/cert.pem --ssl-ca-certs /usr/local/share/ca-certificates/institute_ca.crt
 ```
 
 Consider running with nohup or similar.
@@ -62,7 +72,7 @@ Some notes on arguments:
 
 --host: 0.0.0.0 = bind to all network interfaces. Reliable but greedy in some situations
 
---log-config: Refers to a JSON file for python logging library. An example file is found in /server/logging.json. Uvicorn provides its own logging configuration via `uvicorn.access` and `uvicorn.error`. These may behave undesirably, and can be overridden in the JSON file with an alternate config. Likewise, fastapi logs to `fastapi` if that needs filtering. For logging to files, set `use_colors = False` in the relevant handlers or shell colour settings will appear as garbage in the logs.
+--log-config: Refers to a JSON file for python logging library. An example file is found in /src/logging.json. Uvicorn provides its own logging configuration via `uvicorn.access` and `uvicorn.error`. These may behave undesirably, and can be overridden in the JSON file with an alternate config. Likewise, fastapi logs to `fastapi` if that needs filtering. For logging to files, set `use_colors = False` in the relevant handlers or shell colour settings will appear as garbage in the logs.
 
 --ssl-keyfile: A PEM format key for the server certificate
 --ssl-certfile: A PEM format certificate for signing HTTPS communications
@@ -77,11 +87,11 @@ pip install -e .[test]
 pytest
 ```
 
-Individual tests are run in the form `pytest server/tests/init_test.py`
+Individual tests are run in the form `pytest tests/init_test.py`
 
 ### Fixtures
 
-Fixtures reside under `server/tests/fixtures` and are registered in `server/tests/conftest.py`
+Fixtures reside under `tests/fixtures` and are registered in `tests/conftest.py`
 They can also be listed by invoking `pytest --fixtures`
 
 Any fixtures that are not imported in `conftest.py` will not be detected.
@@ -106,7 +116,7 @@ The SET command ensures that the new schema is visible _for one session only_ in
 DB=npg_porch
 export DB_URL=postgresql+psycopg2://npg_admin:$PASS@npg_porch_db:$PORT/$DB
 # note that the script requires a regular PG driver, not the async version showed above
-server/deploy_schema.py
+src/deploy_schema.py
 
 psql --host=npg_porch_db --port=$PORT --username=npg_admin --password -d $DB
 ```

docker/Dockerfile.dev

@@ -0,0 +1,99 @@
+
+ARG BASE_IMAGE=python:3.10-slim
+FROM $BASE_IMAGE as builder
+
+ARG DEBIAN_FRONTEND="noninteractive"
+
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    build-essential \
+    gcc \
+    libsqlite3-dev \
+    unattended-upgrades && \
+    unattended-upgrade -v
+
+WORKDIR /app
+
+COPY .. .
+
+RUN python -m venv /app && \
+    . ./bin/activate && \
+    pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir .
+
+
+FROM $BASE_IMAGE
+
+ARG DEBIAN_FRONTEND
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -q -y --no-install-recommends \
+    libsqlite3-0 \
+    postgresql \
+    sudo \
+    tini \
+    locales && \
+    locale-gen en_GB en_GB.UTF-8 && \
+    localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8
+
+RUN apt-get install -q -y --no-install-recommends \
+    unattended-upgrades && \
+    unattended-upgrade -v && \
+    apt-get remove -q -y unattended-upgrades && \
+    apt-get autoremove -q -y && \
+    apt-get clean -q -y && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8 \
+    TZ="Etc/UTC"
+
+ARG APP_USER=appuser
+ARG APP_UID=1000
+ARG APP_GID=$APP_UID
+
+WORKDIR /app
+
+RUN groupadd --gid $APP_GID $APP_USER && \
+    useradd --uid $APP_UID --gid $APP_GID --shell /bin/bash --create-home $APP_USER
+
+COPY --from=builder --chown=$APP_USER:$APP_GID /app /app
+
+ARG DB_HOST=localhost
+ARG DB_PORT=5432
+ARG DB_SCHEMA=porch_dev
+ARG DB_NAME=porch_dev_db
+ARG DB_USER=porch_admin
+ARG DB_PASS=porch
+ARG URL_SLUG="$DB_USER:$DB_PASS@$DB_HOST:$DB_PORT/$DB_NAME"
+
+ENV DB_HOST=$DB_HOST \
+    DB_PORT=$DB_PORT \
+    DB_SCHEMA=$DB_SCHEMA \
+    DB_NAME=$DB_NAME \
+    DB_USER=$DB_USER \
+    DB_PASS=$DB_PASS \
+    DB_URL="postgresql+psycopg2://$URL_SLUG"
+
+RUN service postgresql start && \
+    /app/docker/scripts/create_database.sh && \
+    /app/docker/scripts/configure_database_service.sh && \
+    . /app/bin/activate && \
+    /app/scripts/deploy_schema.py && \
+    /app/docker/scripts/insert_admin_token.sh && \
+    service postgresql stop
+
+USER $APP_USER
+
+ARG PORT=8081
+
+ENV DB_URL="postgresql+asyncpg://$URL_SLUG" \
+    PORT=${PORT}
+
+EXPOSE ${PORT}
+
+ENTRYPOINT ["/usr/bin/tini", "--"]
+
+CMD ["/app/docker/scripts/entrypoint.sh"]

docker/README.md

@@ -0,0 +1,19 @@
+# Development Dockerfile
+
+The Dockerfile and scripts in this directory may be use to create a development image
+that hosts both the PostgreSQL database and the npg_porch server.
+
+The application is populated with a hard-coded administrator user, password and
+administration token and is configured log to STDERR and STDOUT.
+
+To create an image using the Dockerfile, run the following command from the root of the
+repository:
+
+```bash
+docker build --rm -f docker/Dockerfile.dev -t npg_porch_dev .
+```
+
+The Dockerfile supports a number of arguments that can be passed to the `docker build`
+command to configure most aspects of the application, including user names, passwords,
+database names and ports. However, the default values should be suitable for most
+needs.

docker/logging.json

@@ -0,0 +1,51 @@
+{
+    "version": 1,
+    "formatters": {
+        "default": {
+            "()": "uvicorn.logging.DefaultFormatter",
+            "fmt": "%(levelprefix)s %(message)s",
+            "use_colors": null
+        },
+        "access": {
+            "()": "uvicorn.logging.AccessFormatter",
+            "fmt": "%(levelprefix)s %(client_addr)s - \"%(request_line)s\" %(status_code)s"
+        }
+    },
+    "handlers": {
+        "stderr": {
+            "formatter": "default",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stderr"
+        },
+        "stdout": {
+            "formatter": "default",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stdout"
+        },
+        "access": {
+            "formatter": "access",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stdout"
+        }
+    },
+    "loggers": {
+        "uvicorn": {
+            "handlers": ["access"],
+            "level": "INFO",
+            "propagate": false
+        },
+        "uvicorn.error": {
+            "handlers": ["stderr"],
+            "level": "DEBUG",
+            "propagate": false
+        },
+        "fastapi": {
+            "handlers": ["stderr"],
+            "level": "INFO"
+        }
+    },
+    "root": {
+        "handlers": ["stdout"],
+        "level": "DEBUG"
+    }
+}

docker/scripts/configure_database_service.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eo pipefail
+set -x
+
+APP_USER=${APP_USER:? The APP_USER environment variable must be set}
+
+cat > "/etc/sudoers.d/$APP_USER" << EOF
+$APP_USER ALL= NOPASSWD: /usr/sbin/service postgresql start
+$APP_USER ALL= NOPASSWD: /usr/sbin/service postgresql restart
+$APP_USER ALL= NOPASSWD: /usr/sbin/service postgresql stop
+EOF

docker/scripts/create_database.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eo pipefail
+set -x
+
+pg_isready --quiet || {
+ echo "PostgreSQL is not ready" >&2
+ exit 1
+}
+
+DB_SCHEMA=${DB_SCHEMA:? The DB_SCHEMA environment variable must be set}
+DB_NAME=${DB_NAME:? The DB_NAME environment variable must be set}
+DB_USER=${DB_USER:? The DB_USER environment variable must be set}
+DB_PASS=${DB_PASS:? The DB_PASS environment variable must be set}
+
+sudo -u postgres createuser -D -R -S ${DB_USER}
+sudo -u postgres createdb -O ${DB_USER} ${DB_NAME}
+
+sudo -u postgres psql -d ${DB_NAME} << EOF
+ALTER USER ${DB_USER} WITH PASSWORD '${DB_PASS}';
+
+CREATE SCHEMA ${DB_SCHEMA};
+
+SET search_path TO ${DB_SCHEMA}, public;
+
+GRANT ALL PRIVILEGES ON SCHEMA ${DB_SCHEMA} TO ${DB_USER};
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${DB_SCHEMA} TO ${DB_USER};
+GRANT USAGE ON ALL SEQUENCES IN SCHEMA ${DB_SCHEMA} TO ${DB_USER};
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${DB_SCHEMA} TO ${DB_USER};
+EOF

docker/scripts/entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eo pipefail
+
+sudo service postgresql start
+
+pg_isready --quiet --timeout=30 || {
+ echo "PostgreSQL is not ready" >&2
+ exit 1
+}
+
+PORT=${PORT:? The PORT environment variable must be set}
+
+source /app/bin/activate
+
+uvicorn npg_porch.server:app --host 0.0.0.0 --port ${PORT} --reload --log-config /app/docker/logging.json

docker/scripts/insert_admin_token.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -eo pipefail
+set -x
+
+pg_isready --quiet || {
+ echo "PostgreSQL is not ready" >&2
+ exit 1
+}
+
+DB_SCHEMA=${DB_SCHEMA:? The DB_SCHEMA environment variable must be set}
+DB_NAME=${DB_NAME:? The DB_NAME environment variable must be set}
+
+ADMIN_TOKEN=${ADMIN_TOKEN:="00000000000000000000000000000000"}
+
+sudo -u postgres psql -d ${DB_NAME} << EOF
+INSERT INTO ${DB_SCHEMA}."token" (token, description, date_issued) VALUES ('${ADMIN_TOKEN}', 'Admin token', NOW());
+EOF