Add runner, update model logic and API endpoints

- runner
  - python script to poll backend for jobs, download a job, run analysis, upload results
  - packaged as a miniconda docker image
  - docker-compose.yml to run multiple replicas of the image
- backend
  - remove no longer needed api endpoints and logic
  - update logic of remaining endpoints
  - add new admin endpoints
    - `/api/admin/enable_user`
    - `/api/admin/disable_user`
  - add new runner endpoints
    - `/api/runner/request_job`
    - `/api/runner/input_file`
    - `/api/runner/result`
- frontend
  - update text and fields
  - remove old sample options, add new ones
  - admins can enable/disable users

.gitignore

@@ -54,7 +54,7 @@ share/python-wheels/
 MANIFEST
 
 # PyInstaller
-#  Usually these files are written by a python script from a template
+#  Usually these files are written by a python scripts from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec

backend/pyproject.toml

@@ -30,7 +30,7 @@ dependencies = [
 predicTCR_server = "predicTCR_server.main:main"
 
 [project.optional-dependencies]
-tests = ["pytest", "freezegun"]
+tests = ["pytest"]
 
 [tool.setuptools.dynamic]
 version = { attr = "predicTCR_server.__version__" }

backend/src/predicTCR_server/__init__.py

@@ -1,5 +1,5 @@
-__version__ = "0.0.1"
-
 from predicTCR_server.app import create_app
 
+__version__ = "0.0.1"
+
 __all__ = [create_app]

backend/src/predicTCR_server/app.py

@@ -2,7 +2,6 @@
 
 import os
 import secrets
-import pathlib
 import datetime
 import flask
 from flask import Flask
@@ -19,17 +18,15 @@
     Sample,
     User,
     add_new_user,
+    add_new_runner_user,
     reset_user_password,
+    enable_user,
     activate_user,
     add_new_sample,
     get_samples,
-    remaining_samples_this_week,
-    get_current_settings,
-    set_current_settings,
-    update_samples_zipfile,
-    process_result,
     send_password_reset_email,
-    resubmit_sample,
+    request_job,
+    process_result,
 )
 
 
@@ -50,9 +47,9 @@ def create_app(data_path: str = "/predictcr_data"):
     app.config["JWT_ACCESS_TOKEN_EXPIRES"] = datetime.timedelta(minutes=60)
     app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{data_path}/predicTCR.db"
     app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
-    # limit max file upload size to 384mb
-    app.config["MAX_CONTENT_LENGTH"] = 384 * 1024 * 1024
-    app.config["CIRCUITSEQ_DATA_PATH"] = data_path
+    # limit max file upload size to 20mb
+    app.config["MAX_CONTENT_LENGTH"] = 20 * 1024 * 1024
+    app.config["PREDICTCR_DATA_PATH"] = data_path
 
     CORS(app)
 
@@ -86,6 +83,9 @@ def login():
         if not user.activated:
             logger.info("  -> user not activated")
             return jsonify(message="User account is not yet activated"), 400
+        if not user.enabled:
+            logger.info("  -> user not enabled")
+            return jsonify(message="User account is not yet enabled"), 400
         if not user.check_password(password):
             logger.info("  -> wrong password")
             return jsonify(message="Incorrect password"), 400
@@ -142,71 +142,49 @@ def change_password():
             400,
         )
 
-    @app.route("/api/remaining", methods=["GET"])
-    def remaining():
-        return remaining_samples_this_week()
-
-    @app.route("/api/running_options", methods=["GET"])
-    @jwt_required()
-    def running_options():
-        settings = get_current_settings()
-        return jsonify(running_options=settings["running_options"])
-
     @app.route("/api/samples", methods=["GET"])
     @jwt_required()
     def samples():
         return get_samples(current_user.email)
 
-    @app.route("/api/reference_sequence", methods=["POST"])
+    @app.route("/api/input_file", methods=["POST"])
     @jwt_required()
-    def reference_sequence():
-        primary_key = request.json.get("primary_key", None)
+    def input_file():
+        sample_id = request.json.get("sample_id", None)
         logger.info(
-            f"User {current_user.email} requesting reference sequence with key {primary_key}"
+            f"User {current_user.email} requesting results for sample {sample_id}"
         )
-        filters = {"primary_key": primary_key}
-        if not current_user.is_admin:
+        filters = {"id": sample_id}
+        if not current_user.is_admin and not current_user.is_runner:
             filters["email"] = current_user.email
         user_sample = db.session.execute(
             db.select(Sample).filter_by(**filters)
         ).scalar_one_or_none()
         if user_sample is None:
-            logger.info(f"  -> sample with key {primary_key} not found")
+            logger.info(f"  -> sample {sample_id} not found")
             return jsonify(message="Sample not found"), 400
-        if not user_sample.has_reference_seq_zip:
-            logger.info(
-                f"  -> sample with key {primary_key} found but does not contain a reference sequence"
-            )
-            return jsonify(message="Sample does not contain a reference sequence"), 400
-        requested_file = pathlib.Path(user_sample.reference_seq_zip_path())
-        if not requested_file.is_file():
-            logger.info(f"  -> file {requested_file} not found")
-            return jsonify(message="Reference sequence file not found"), 400
-        logger.info(f"Returning file {user_sample.reference_seq_zip_path()}")
-        return flask.send_file(user_sample.reference_seq_zip_path(), as_attachment=True)
+        return flask.send_file(user_sample.input_file_path(), as_attachment=True)
 
     @app.route("/api/result", methods=["POST"])
     @jwt_required()
     def result():
-        primary_key = request.json.get("primary_key", None)
+        sample_id = request.json.get("sample_id", None)
         logger.info(
-            f"User {current_user.email} requesting results for key {primary_key}"
+            f"User {current_user.email} requesting results for sample {sample_id}"
         )
-        filters = {"primary_key": primary_key}
+        filters = {"id": sample_id}
         if not current_user.is_admin:
             filters["email"] = current_user.email
         user_sample = db.session.execute(
             db.select(Sample).filter_by(**filters)
         ).scalar_one_or_none()
         if user_sample is None:
-            logger.info(f"  -> sample with key {primary_key} not found")
+            logger.info(f"  -> sample {sample_id} not found")
             return jsonify(message="Sample not found"), 400
         if not user_sample.has_results_zip:
-            logger.info(
-                f"  -> sample with key {primary_key} found but no results available"
-            )
+            logger.info(f"  -> sample {sample_id} found but no results available")
             return jsonify(message="No results available"), 400
-        requested_file = pathlib.Path(user_sample.results_file_path())
+        requested_file = user_sample.result_file_path()
         if not requested_file.is_file():
             logger.info(f"  -> file {requested_file} not found")
             return jsonify(message="Results file not found"), 400
@@ -219,84 +197,90 @@ def add_sample():
         email = current_user.email
         form_as_dict = request.form.to_dict()
         name = form_as_dict.get("name", "")
-        running_option = form_as_dict.get("running_option", "")
-        concentration = int(form_as_dict.get("concentration", "0"))
-        reference_sequence_files = request.files.getlist("file")
+        tumor_type = form_as_dict.get("tumor_type", "")
+        source = form_as_dict.get("source", "")
+        infile = request.files.get("file")
         logger.info(f"Adding sample {name} from {email}")
         new_sample, error_message = add_new_sample(
-            email, name, running_option, concentration, reference_sequence_files
+            email, name, tumor_type, source, infile
         )
         if new_sample is not None:
             logger.info("  - > success")
             return jsonify(sample=new_sample)
         return jsonify(message=error_message), 400
 
-    @app.route("/api/admin/settings", methods=["GET", "POST"])
-    @jwt_required()
-    def admin_settings():
-        if not current_user.is_admin:
-            return jsonify(message="Admin account required"), 400
-        if flask.request.method == "POST":
-            message, code = set_current_settings(current_user.email, request.json)
-            return jsonify(message=message), code
-        else:
-            return get_current_settings()
-
     @app.route("/api/admin/samples", methods=["GET"])
     @jwt_required()
     def admin_all_samples():
         if not current_user.is_admin:
             return jsonify(message="Admin account required"), 400
         return jsonify(get_samples())
 
-    @app.route("/api/admin/resubmit_sample", methods=["POST"])
+    @app.route("/api/admin/enable_user", methods=["POST"])
     @jwt_required()
-    def admin_resubmit_sample():
+    def admin_enable_user():
         if not current_user.is_admin:
             return jsonify(message="Admin account required"), 400
-        primary_key = request.json.get("primary_key", "")
-        message, code = resubmit_sample(primary_key)
+        user_email = request.json.get("user_email", "")
+        message, code = enable_user(user_email, True)
         return jsonify(message=message), code
 
-    @app.route("/api/admin/zipsamples", methods=["POST"])
+    @app.route("/api/admin/disable_user", methods=["POST"])
     @jwt_required()
-    def admin_zip_samples():
+    def admin_disable_user():
         if not current_user.is_admin:
             return jsonify(message="Admin account required"), 400
-        logger.info(
-            f"Request for zipfile of samples from Admin user {current_user.email}"
-        )
-        zip_file = update_samples_zipfile(datetime.date.today())
-        return flask.send_file(zip_file, as_attachment=True)
+        user_email = request.json.get("user_email", "")
+        message, code = enable_user(user_email, False)
+        return jsonify(message=message), code
 
     @app.route("/api/admin/users", methods=["GET"])
     @jwt_required()
     def admin_users():
-        if current_user.is_admin:
-            users = db.session.execute(db.select(User)).scalars().all()
-            return jsonify(users=[user.as_dict() for user in users])
-        return jsonify(message="Admin account required"), 400
-
-    @app.route("/api/admin/token", methods=["GET"])
-    @jwt_required()
-    def admin_token():
-        if current_user.is_admin:
-            access_token = create_access_token(
-                identity=current_user, expires_delta=datetime.timedelta(weeks=26)
-            )
-            return jsonify(access_token=access_token)
-        return jsonify(message="Admin account required"), 400
+        if not current_user.is_admin:
+            return jsonify(message="Admin account required"), 400
+        users = (
+            db.session.execute(db.select(User).order_by(db.desc(User.id)))
+            .scalars()
+            .all()
+        )
+        return jsonify(users=[user.as_dict() for user in users])
 
-    @app.route("/api/admin/result", methods=["POST"])
+    @app.route("/api/admin/runner_token", methods=["GET"])
     @jwt_required()
-    def admin_upload_result():
+    def admin_runner_token():
         if not current_user.is_admin:
             return jsonify(message="Admin account required"), 400
-        email = current_user.email
+        runner_user = add_new_runner_user()
+        if runner_user is None:
+            return jsonify(message="Failed to create runner account"), 500
+        access_token = create_access_token(
+            identity=runner_user, expires_delta=datetime.timedelta(weeks=26)
+        )
+        return jsonify(access_token=access_token)
+
+    @app.route("/api/runner/request_job", methods=["POST"])
+    @jwt_required()
+    def runner_request_job():
+        if not current_user.is_runner:
+            return jsonify(message="Runner account required"), 400
+        runner_hostname = request.json.get("runner_hostname", "")
+        logger.info(f"Runner {current_user.email} / {runner_hostname} requesting job")
+        sample_id = request_job()
+        if sample_id is None:
+            return jsonify(message="No job available"), 204
+        return {"sample_id": sample_id}
+
+    @app.route("/api/runner/result", methods=["POST"])
+    @jwt_required()
+    def runner_result():
+        if not current_user.is_runner:
+            return jsonify(message="Runner account required"), 400
         form_as_dict = request.form.to_dict()
-        primary_key = form_as_dict.get("primary_key", "")
-        success = request.form.to_dict().get("success", None)
-        logger.info(f"Result upload for '{primary_key}' from user {email}")
+        sample_id = form_as_dict.get("sample_id", None)
+        if sample_id is None:
+            return jsonify(message="Missing key: sample_id"), 400
+        success = form_as_dict.get("success", None)
         if success is None or success.lower() not in ["true", "false"]:
             logger.info("  -> missing success key")
             return jsonify(message="Missing key: success=True/False"), 400
@@ -305,7 +289,14 @@ def admin_upload_result():
         if success is True and zipfile is None:
             logger.info("  -> missing zipfile")
             return jsonify(message="Result has success=True but no file"), 400
-        message, code = process_result(primary_key, success, zipfile)
+        runner_hostname = form_as_dict.get("runner_hostname", "")
+        logger.info(
+            f"Result upload for '{sample_id}' from runner {current_user.email} / {runner_hostname}"
+        )
+        error_message = form_as_dict.get("error_message", None)
+        if error_message is not None:
+            logger.info(f"  -> error message: {error_message}")
+        message, code = process_result(sample_id, success, zipfile)
         return jsonify(message=message), code
 
     with app.app_context():

backend/src/predicTCR_server/email.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import smtplib
+from email.message import EmailMessage
+from predicTCR_server.settings import predicTCR_url
+
+
+def _wrap_email_message(email: str, message: str) -> str:
+    return f"Dear {email},\n\n{message}\n\nBest wishes,\n\npredicTCR Team.\nhttps://{predicTCR_url}"
+
+
+def _send_email_message(email_message: EmailMessage) -> None:
+    postfix_server_address = "email:587"
+    with smtplib.SMTP(postfix_server_address) as s:
+        s.send_message(email_message)
+
+
+def send_email(email: str, subject: str, message: str) -> None:
+    msg = EmailMessage()
+    msg["From"] = f"no-reply@{predicTCR_url}"
+    msg["To"] = email
+    msg.set_content(_wrap_email_message(email, message))
+    msg["Subject"] = subject
+    _send_email_message(msg)