From c490eec08e3f8e5ec80e9f9f229f0a4db7867482 Mon Sep 17 00:00:00 2001
From: Liam Keegan <liam@keegan.ch>
Date: Thu, 29 Aug 2024 12:28:13 +0200
Subject: [PATCH] Improve development environment and update readmes

- backend
  - only enable CORS for /runner endpoints
  - enable CORS for everything in development server
  - log email messages instead of sending them in development server
- frontend
  - add .env.development with default backend development server api url
- update docs
---
 README.md                              | 36 +++++++--------
 README_DEPLOYMENT.md                   |  4 +-
 README_DEV.md                          | 62 +++++++++++++++-----------
 backend/src/predicTCR_server/app.py    |  8 ++--
 backend/src/predicTCR_server/logger.py |  2 +-
 backend/src/predicTCR_server/main.py   | 10 +++++
 backend/src/predicTCR_server/model.py  |  2 +-
 backend/src/predicTCR_server/utils.py  |  2 +-
 frontend/.env.development              |  1 +
 runner/README.md                       |  9 ++--
 runner/docker-compose.yml              |  4 +-
 runner/src/predicTCR_runner/runner.py  |  2 +-
 12 files changed, 77 insertions(+), 65 deletions(-)
 create mode 100644 frontend/.env.development

diff --git a/README.md b/README.md
index b8ebd06..6d2b610 100644
--- a/README.md
+++ b/README.md
@@ -11,34 +11,28 @@ The source code for the [predicTCR website](https://predictcr.lkeegan.dev/).
 Users of the site can
 
 - sign up with a valid email address
-- request a sample, optionally providing a reference sequence
+- upload a sample to be analyzed
 - see a list of their requested samples
-- download their reference sequences
-- download full analysis data
-- automatically receive an email with their results
+- download the analysis results
+- TODO automatically receive an email with their results
 
 ### Admins
 
 Users with admin rights can additionally
 
 - see a list of all users
-- see a list of all requested samples and results
-- change the site settings
-  - set which day of the week sample submission closes
-  - set number of plate rows/cols
-  - add/remove running options
-- download a zipped tsv of requests from the current week
-  - this includes the reference sequences as fasta files
-- upload a zipfile with successful analysis results to be sent to the user
-- upload unsuccessful analysis result status to be sent to the user
-
-### REST API
-
-Admins can also generate an API token,
-then do all of the above programatically using
-the provided REST API:
-
-- [api_examples.ipynb](https://github.com/ssciwr/predicTCR/blob/main/notebooks/api_examples.ipynb)
+- see a list of all samples and results
+- create an API token for a runner to use
+
+## Runners
+
+The analysis of the samples is done by runners, which
+
+- are a separate service packaged as a docker container
+- can be run on any machine with docker installed
+- authenticate with the API using a token
+- regularly check for new samples to analyze
+- do sample analysis and upload the results
 
 ## Developer info
 
diff --git a/README_DEPLOYMENT.md b/README_DEPLOYMENT.md
index abffaea..2477522 100644
--- a/README_DEPLOYMENT.md
+++ b/README_DEPLOYMENT.md
@@ -34,7 +34,7 @@ sudo docker compose logs
 
 ### SSL certificate
 
-To generate initial SSL certificates for domain `domain.com` on the VM:
+To generate SSL certificates for domain `domain.com` from [Let's Encrypt](https://letsencrypt.org/) using [Certbot](https://certbot.eff.org/):
 
 ```
 sudo docker run -it --rm --name certbot -v "/etc/letsencrypt:/etc/letsencrypt" -v "/var/lib/letsencrypt:/var/lib/letsencrypt" -p80:80 -p443:443 certbot/certbot certonly -d domain.com
@@ -59,4 +59,4 @@ cd predicTCR
 sudo sqlite3 docker_volume/predicTCR.db
 sqlite> UPDATE user SET is_admin=true WHERE email='user@embl.de';
 sqlite> .quit
-```
+```
\ No newline at end of file
diff --git a/README_DEV.md b/README_DEV.md
index c3dc188..33f3c65 100644
--- a/README_DEV.md
+++ b/README_DEV.md
@@ -1,8 +1,16 @@
 # predicTCR website developer info
 
-Some information on how to locally build and deploy the website if you would like to make changes to the code.
+Some information on how to locally build and serve the website if you would like to make changes to the code.
+There are two ways to do this:
 
-## Run locally with docker compose
+- docker
+  - closer to production environment
+  - but less convenient for development - you need to rebuild the image every time you make a change
+- python/pnpm
+  - further from production environment setup
+  - but convenient for development - see changes immediately without having to rebuild or restart anything
+
+## Run locally with docker
 
 Requires docker and docker compose.
 
@@ -28,12 +36,6 @@ docker compose up --build
 The website is then served at https://localhost/
 (note that the SSL keys are self-signed keys and your browser will still warn about the site being insecure.)
 
-### SSL
-
-SSL cert/key by default are assumed to exist as `cert.pem` and `key.pem`
-in the folder where you run the docker compose command.
-To point to different files, set the `PREDICTCR_SSL_CERT` and `PREDICTCR_SSL_KEY` environment variables.
-
 ### Database
 
 The database will by default be stored in a `docker_volume` folder
@@ -49,20 +51,33 @@ If this is not set or is less than 16 chars, a new random secret key is generate
 ### User signup activation email
 
 When you sign up for an account when running locally it will send an email (if port 25 is open) to whatever address you use.
-If the port is blocked you can see the activation_token in the docker logs,
+You can also see the activation_token in the docker logs,
 and activate your local account by going to `https://localhost/activate/<activation_token_from_logs>`
-To make yourself an admin user, see the production deployment section below.
-
-### Admin user
 
-To make an existing user with email `user@embl.de` into an admin, shutdown the docker containers if runner, then
+### Make yourself an admin user
 
 ```
 sudo sqlite3 docker_volume/predicTCR.db
-sqlite> UPDATE user SET is_admin=true WHERE email='user@embl.de';
+sqlite> UPDATE user SET is_admin=true WHERE email='you@address.com';
 sqlite> .quit
 ```
 
+### Start a runner
+
+In the runner directory, create a `.env` file with the following content:
+
+```
+PREDICTCR_RUNNER_API_URL="http://backend:8080/api"
+PREDICTCR_RUNNER_JWT_TOKEN="" # you need to generate this using the admin page of your local instance
+PREDICTCR_RUNNER_LOG_LEVEL=DEBUG
+```
+
+Then build and start the runner with:
+
+```sh
+docker compose up --build
+```
+
 ## Run locally with Python and pnpm
 
 Requires Python and [pnpm](https://pnpm.io/installation#using-a-standalone-script)
@@ -94,17 +109,10 @@ pnpm run dev
 The website is then served at http://localhost:5173/.
 Note that the email activation message will be written to the console instead of being sent by email.
 
-## Implementation
-
-### Backend
-
-The backend is a Python Flask REST API, see [backend/README.md](backend/README.md) for more details.
-
-### Frontend
+4. install and run the runner:
 
-The frontend is a vue.js app, see [frontend/README.md](frontend/README.md) for more details.
-
-### Docker
-
-Both the backend and the frontend have a Dockerfile,
-and there is a docker compose file to coordinate them.
+```sh
+cd runner
+pip install .
+predicTCR_runner
+```
diff --git a/backend/src/predicTCR_server/app.py b/backend/src/predicTCR_server/app.py
index f47e2c2..a833586 100644
--- a/backend/src/predicTCR_server/app.py
+++ b/backend/src/predicTCR_server/app.py
@@ -11,7 +11,7 @@
 from flask_jwt_extended import current_user
 from flask_jwt_extended import jwt_required
 from flask_jwt_extended import JWTManager
-from flask_cors import CORS
+from flask_cors import cross_origin
 from predicTCR_server.logger import get_logger
 from predicTCR_server.model import (
     db,
@@ -31,7 +31,7 @@
 
 
 def create_app(data_path: str = "/predictcr_data"):
-    logger = get_logger("predicTCRServer")
+    logger = get_logger()
     app = Flask("predicTCRServer")
     jwt_secret_key = os.environ.get("JWT_SECRET_KEY")
     if jwt_secret_key is not None and len(jwt_secret_key) > 16:
@@ -51,8 +51,6 @@ def create_app(data_path: str = "/predictcr_data"):
     app.config["MAX_CONTENT_LENGTH"] = 100 * 1024 * 1024
     app.config["PREDICTCR_DATA_PATH"] = data_path
 
-    CORS(app)
-
     jwt = JWTManager(app)
     db.init_app(app)
 
@@ -279,6 +277,7 @@ def admin_runner_token():
         return jsonify(access_token=access_token)
 
     @app.route("/api/runner/request_job", methods=["POST"])
+    @cross_origin()
     @jwt_required()
     def runner_request_job():
         if not current_user.is_runner:
@@ -291,6 +290,7 @@ def runner_request_job():
         return {"sample_id": sample_id}
 
     @app.route("/api/runner/result", methods=["POST"])
+    @cross_origin()
     @jwt_required()
     def runner_result():
         if not current_user.is_runner:
diff --git a/backend/src/predicTCR_server/logger.py b/backend/src/predicTCR_server/logger.py
index 930c7cb..528015b 100644
--- a/backend/src/predicTCR_server/logger.py
+++ b/backend/src/predicTCR_server/logger.py
@@ -2,7 +2,7 @@
 import logging
 
 
-def get_logger(name: str):
+def get_logger(name: str = "predicTCRServer") -> logging.Logger:
     logger = logging.getLogger(name)
     if not logger.handlers:
         logger.setLevel(logging.DEBUG)
diff --git a/backend/src/predicTCR_server/main.py b/backend/src/predicTCR_server/main.py
index 89df321..830d0ba 100644
--- a/backend/src/predicTCR_server/main.py
+++ b/backend/src/predicTCR_server/main.py
@@ -1,6 +1,9 @@
 from __future__ import annotations
 import click
 from predicTCR_server import create_app
+from predicTCR_server.logger import get_logger
+from flask_cors import CORS
+import predicTCR_server.email
 
 
 @click.command()
@@ -9,6 +12,13 @@
 @click.option("--data-path", default=".", show_default=True)
 def main(host: str, port: int, data_path: str):
     app = create_app(data_path=data_path)
+    # local development server: enable CORS on all routes for all origins
+    CORS(app)
+    # local development server: log email messages instead of sending them
+    logger = get_logger()
+    predicTCR_server.email._send_email_message = lambda email_message: logger.info(
+        email_message
+    )
     app.run(host=host, port=port)
 
 
diff --git a/backend/src/predicTCR_server/model.py b/backend/src/predicTCR_server/model.py
index df5be0e..2252bd4 100644
--- a/backend/src/predicTCR_server/model.py
+++ b/backend/src/predicTCR_server/model.py
@@ -25,7 +25,7 @@
 
 db = SQLAlchemy()
 ph = argon2.PasswordHasher()
-logger = get_logger("predicTCRServer")
+logger = get_logger()
 
 
 class Status(str, Enum):
diff --git a/backend/src/predicTCR_server/utils.py b/backend/src/predicTCR_server/utils.py
index deacbb9..d4f9bf1 100644
--- a/backend/src/predicTCR_server/utils.py
+++ b/backend/src/predicTCR_server/utils.py
@@ -4,7 +4,7 @@
 from itsdangerous.url_safe import URLSafeTimedSerializer
 from datetime import datetime
 
-logger = get_logger("predicTCRServer")
+logger = get_logger()
 
 
 def timestamp_now() -> int:
diff --git a/frontend/.env.development b/frontend/.env.development
new file mode 100644
index 0000000..0b7c683
--- /dev/null
+++ b/frontend/.env.development
@@ -0,0 +1 @@
+VITE_REST_API_LOCATION=http://localhost:8080/api
diff --git a/runner/README.md b/runner/README.md
index 65fad5c..cc5962e 100644
--- a/runner/README.md
+++ b/runner/README.md
@@ -27,14 +27,15 @@ PREDICTCR_RUNNERS=4
 PREDICTCR_POLL_INTERVAL=5
 ```
 
-With this .env file, `docker compose up -d` will start 4 runner images in the background, which poll the web service for new jobs every 5 seconds.
+With this .env file, `docker compose up -d` will start 4 runner images in the background,
+which poll the web service for new jobs every 5 seconds.
 
 ## Development
 
 To test locally using Docker, you can directly talk to the backend service (this works because both docker-compose files use the same docker network)
 
 ```
-PREDICTCR_API_URL="http://backend:8080/api"
-PREDICTCR_JWT_TOKEN="" # you need to generate this using the admin page of your local instance
-PREDICTCR_LOG_LEVEL=DEBUG
+PREDICTCR_RUNNER_API_URL="http://backend:8080/api"
+PREDICTCR_RUNNER_JWT_TOKEN="" # you need to generate this using the admin page of your local instance
+PREDICTCR_RUNNER_LOG_LEVEL=DEBUG
 ```
diff --git a/runner/docker-compose.yml b/runner/docker-compose.yml
index 16b070d..2a8778b 100644
--- a/runner/docker-compose.yml
+++ b/runner/docker-compose.yml
@@ -2,8 +2,6 @@ services:
   runner:
     image: ghcr.io/ssciwr/predictcr_runner:${PREDICTCR_DOCKER_IMAGE_TAG:-latest}
     build: .
-    volumes:
-      - ${PREDICTCR_RUNNER_DATA_DIR:-./data}:/data
     environment:
       - PREDICTCR_API_URL=${PREDICTCR_API_URL:-https://predictcr.lkeegan.dev/api}
       - PREDICTCR_JWT_TOKEN=${PREDICTCR_JWT_TOKEN:-}
@@ -11,7 +9,7 @@ services:
       - PREDICTCR_LOG_LEVEL=${PREDICTCR_LOG_LEVEL:-INFO}
     deploy:
       mode: replicated
-      replicas: ${PREDICTCR_RUNNERS:-1}
+      replicas: ${PREDICTCR_RUNNER_JOBS:-1}
     networks:
       - predictcr-network
 
diff --git a/runner/src/predicTCR_runner/runner.py b/runner/src/predicTCR_runner/runner.py
index cbc2f68..f10f3e4 100644
--- a/runner/src/predicTCR_runner/runner.py
+++ b/runner/src/predicTCR_runner/runner.py
@@ -120,4 +120,4 @@ def start(self):
             if job_id is not None:
                 self._run_job(job_id)
             else:
-                time.sleep(self.poll_interval)
+                time.sleep(self.poll_interval)
\ No newline at end of file