Merge branch 'staging'

.github/workflows/ci.yaml

@@ -31,7 +31,7 @@ on:
 
 jobs:
   CI:
-    if: github.repository == 'scilifelabdatacentre/stackn'
+    if: github.repository == 'scilifelabdatacentre/serve'
     runs-on: ubuntu-20.04
     env:
       working-directory: .

.github/workflows/e2e-tests.yaml

@@ -26,7 +26,7 @@ on:
 
 jobs:
   e2e:
-    if: github.repository == 'scilifelabdatacentre/stackn'
+    if: github.repository == 'scilifelabdatacentre/serve'
     runs-on: ubuntu-20.04
     env:
       working-directory: .
@@ -121,7 +121,7 @@ jobs:
 
 
       - name: Cypress run e2e tests
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@v6
         with:
           working-directory: ${{env.working-directory}}
           config: pageLoadTimeout=100000,baseUrl=${{ env.STUDIO_URL }}
@@ -147,4 +147,4 @@ jobs:
           ## limits ssh access and adds the ssh public key for the user which triggered the workflow
           limit-access-to-actor: true
           ## limits ssh access and adds the ssh public keys of the listed GitHub users
-          limit-access-to-users: sandstromviktor,churnikov,hamzaimran08,alfredeen,akochari
+          limit-access-to-users: churnikov,hamzaimran08,alfredeen,akochari,anondo1969

.github/workflows/publish.yaml

@@ -26,7 +26,7 @@ on:
 jobs:
   publish-on-success:
     if: |
-      github.repository == 'scilifelabdatacentre/stackn' &&
+      github.repository == 'scilifelabdatacentre/serve' &&
       (
         (github.event_name == 'push' && contains(github.ref, 'refs/tags/'))
         ||

.gitignore

@@ -26,6 +26,7 @@ downloads/
 eggs/
 .eggs/
 lib/
+!static/tagulous/lib/
 lib64/
 parts/
 sdist/
@@ -154,3 +155,5 @@ node_modules
 # The media folder contents
 media/*
 !media/.gitkeep
+
+local_.dockerfile

Dockerfile.cypress

@@ -1,4 +1,4 @@
-FROM cypress/included:11.2.0
+FROM cypress/included:13.3.0
 
 WORKDIR /app
 

README.md

@@ -6,21 +6,21 @@
       <img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg">
    </a>
    <br />
-      <img alt="Pre-commit" src="https://github.com/ScilifelabDataCentre/stackn/actions/workflows/pre-commit.yaml/badge.svg?branch=develop">
-      <img alt="CI" src="https://github.com/ScilifelabDataCentre/stackn/actions/workflows/ci.yaml/badge.svg?branch=develop">
-      <img alt="End2end tests" src="https://github.com/ScilifelabDataCentre/stackn/actions/workflows/e2e-tests.yaml/badge.svg?branch=develop">
+      <img alt="Pre-commit" src="https://github.com/ScilifelabDataCentre/serve/actions/workflows/pre-commit.yaml/badge.svg?branch=develop">
+      <img alt="CI" src="https://github.com/ScilifelabDataCentre/serve/actions/workflows/ci.yaml/badge.svg?branch=develop">
+      <img alt="End2end tests" src="https://github.com/ScilifelabDataCentre/serve/actions/workflows/e2e-tests.yaml/badge.svg?branch=develop">
 
 </p>
 
 # SciLifeLab Serve
 
 SciLifeLab Serve ([https://serve.scilifelab.se](https://serve.scilifelab.se)) is a platform offering machine learning model serving, app hosting (Shiny, Streamlit, Dash, etc.), web-based integrated development environments, and other tools to life science researchers affiliated with a Swedish research institute. It is developed and operated by the [SciLifeLab Data Centre](https://github.com/ScilifelabDataCentre), part of [SciLifeLab](https://scilifelab.se/). See [this page for information about funders and mandate](https://serve.scilifelab.se/about/).
 
-This repository contains source code for SciLifeLab Serve. It is  based on the open-source platform [Stackn](https://github.com/scaleoutsystems/stackn).
+This repository contains source code for the main Django application of SciLifeLab Serve.
 
 ## Reporting bugs and requesting features
 
-If you are using SciLifeLab Serve and notice a bug or if there is a feature you would like to be added feel free to [create an issue](https://github.com/ScilifelabDataCentre/stackn/issues/new/choose) with a bug report or feature request.
+If you are using SciLifeLab Serve and notice a bug or if there is a feature you would like to be added feel free to [create an issue](https://github.com/ScilifelabDataCentre/serve/issues/new/choose) with a bug report or feature request.
 
 ## Development and contributions
 
@@ -30,14 +30,20 @@ We welcome contributions to the code. When you want to make a contribution pleas
 
 The  `main` branch contains code behind the version that is currently deployed in production - https://serve.scilifelab.se. The branches `develop` and `staging` contain current development versions at different stages.
 
+### Local development setup
+
+There are multiple ways to set up a local development environment for SciLifeLab Serve. Below you can find instructions on how to set up a local development environment using Docker Compose or Rancher Desktop.
+
+Both have their pros and cons. Docker Compose is easier to set up but does not provide a full Kubernetes environment. Rancher Desktop provides a full Kubernetes environment but is more complex to set up.
+
 ### Deploy Serve for local development with Docker Compose
 
 It is possible to deploy and work with the user interface of Serve without a running Kubernetes cluster, in that case you can skip the step related to that below. However, in order to be able to deploy and modify resources (apps, notebooks, persistent storage, etc.) a running local cluster is required; it can be created for example with [microk8s](https://microk8s.io/).
 
 1. Clone this repository locally:
 ```
-$ git clone https://github.com/ScilifelabDataCentre/stackn
-$ cd stackn
+$ git clone https://github.com/ScilifelabDataCentre/serve
+$ cd serve
 ```
 
 2. Create a copy of the .env template file
@@ -75,6 +81,89 @@ This assumes you have the correct ssh key in your ssh-agent. If you like to give
 $ docker compose up -d
 ```
 
+### Deploy Serve for local development with Rancher Desktop
+
+Start with instructions in [Serve Charts > How to Deploy](https://github.com/ScilifelabDataCentre/serve-charts?tab=readme-ov-file#how-to-deploy) and come back here when you get to the point of building the studio image.
+
+Again, this setup assumes you have [Rancher Desktop](https://rancherdesktop.io/) installed and running.
+
+```bash
+$ git clone [email protected]:ScilifelabDataCentre/stackn.git
+$ cd stackn
+$ git checkout develop
+$ cp .env.template .env
+$ cp ~/.kube/config cluster.conf
+$ cp ~/.ssh/id_rsa.pub id_rsa.pub
+$ cat Dockerfile local.Dockerfile > local_.dockerfile
+$ nerdctl build --namespace k8s.io -t mystudio -f local_.dockerfile .
+```
+
+Now continue setting up serve charts until you get to the PyCharm setup.
+
+##### PyCharm setup
+
+> Prerequisite: This setup assumes you have PyCharm Professional installed.
+
+1. Do this weirdness due to [this](https://youtrack.jetbrains.com/issue/PY-55338/Connection-to-python-console-refused-with-docker-interpreter-on-Linux)
+    1. go to Help | Find Action | Registry
+    2. disable python.use.targets.api
+    3. recreate the interpreter from scratch
+2. Setup ssh interpreter
+```bash
+# This will open ssh connection from the pod to our host machine
+# Because we made everything super NOT secure for local development you can ssh there without password and as root
+$ sudo kubectl port-forward svc/serve-studio 22:22
+```
+3. Set up the interpreter in PyCharm
+    1. Go to `PyCharm | Settings | Project: stackn | Python Interpreter`
+    2. Add new interpreter
+    3. Choose SSH
+    4. Host: localhost
+    5. Port: 22
+    6. Username: root
+    7. Auth type: Password
+    8. Password: root
+    9. Interpreter path: /usr/local/bin/python
+    10. Python interpreter path: /usr/local/bin/python
+    11. **Important** Don't accept synchronization option from PyCharm. There is no need for it because we already synchronize the code with the pod using volume mounts provided by Rancher Desktop.
+4. Set up the environment variables
+    1. Go to Run | Edit Configurations
+    2. Add new configuration
+    3. Choose `Django server`
+    4. Name: `Serve`
+    5. Host: `0.0.0.0`
+    6. Port: `8080`
+    7. Click Modify options and select `No reload` and `Environment variables`
+    8. Add environment variables from the studio pod
+    9. Make sure the Working directory is /app.
+    10. The Path mappings should be /path/to/your/stackn=/app.
+
+```bash
+$ kubectl get po
+# find the studio pod serve-studio-123
+$ kubectl exec -it <serve-studio-123> -- /bin/bash
+# Run migrations
+# For simplicity just run sh scripts/run_web.sh
+$ sh scripts/run_web.sh
+# And then stop the process when the server is running
+
+# Now you are in the studio container
+# Type env
+$ env
+# Copy the whole output in to the pycharm environment configuration
+```
+
+Copy environment variables to the PyCharm Django configuration. The environment variables need to be separated by a semi-colon. To achieve this, click on the list icon in the Environment variables input box and then in the popup, click paste.
+
+Make sure that the Django Framework settings in PyCharm are correctly setup.
+To check, go to  PyCharm | Settings | Languages & Frameworks | Django and check the following settings
+- Enable Django Support should be checked.
+- Django project root should be `/path/to/your/stackn`
+- Settings should be `studio/settings.py`
+- Manage script should be `manage.py`
+
+Now that you are done, you can run Django server using PyCharm and access the studio at [http://studio.127.0.0.1.nip.io/](http://studio.127.0.0.1.nip.io/)
+
 ## Contact information
 
 To get in touch with the development team behind SciLifeLab Serve send us an email: [email protected].

api/views.py

@@ -878,7 +878,7 @@ def update_app_status(request: HttpRequest) -> HttpResponse:
 
     # POST verb
     if request.method == "POST":
-        logger.info("API method update_app_status called with POST verb.")
+        logger.debug("API method update_app_status called with POST verb.")
 
         utc = pytz.UTC
 
@@ -891,7 +891,7 @@ def update_app_status(request: HttpRequest) -> HttpResponse:
             new_status = request.data["new-status"]
 
             if len(new_status) > 15:
-                logger.debug("Status code is longer than 15 chars so shortening: %s", new_status)
+                logger.debug(f"Status code is longer than 15 chars so shortening: {new_status}")
                 new_status = new_status[:15]
 
             event_ts = datetime.strptime(request.data["event-ts"], "%Y-%m-%dT%H:%M:%S.%fZ")
@@ -901,19 +901,16 @@ def update_app_status(request: HttpRequest) -> HttpResponse:
             event_msg = request.data.get("event-msg", None)
 
         except KeyError as err:
-            logger.error("API method called with invalid input. Missing required input parameter: %s", err)
+            logger.error(f"API method called with invalid input. Missing required input parameter: {err}")
             return Response(f"Invalid input. Missing required input parameter: {err}", 400)
 
         except Exception as err:
-            logger.error("API method called with invalid input:  %s, %s", err, type(err))
+            logger.error(f"API method called with invalid input: {err}, type: {type(err)}")
             return Response(f"Invalid input. {err}", 400)
 
         logger.debug(
-            "API method update_app_status input: release=%s, new_status=%s, event_ts=%s, event_msg=%s",
-            release,
-            new_status,
-            event_ts,
-            event_msg,
+            f"API method update_app_status input: release={release}, new_status={new_status}, \
+            event_ts={event_ts}, event_msg={event_msg}"
         )
 
         try:
@@ -944,17 +941,16 @@ def update_app_status(request: HttpRequest) -> HttpResponse:
                 )
 
             else:
-                logger.error("Unknown return code from handle_update_status_request() = %s", result, exc_info=True)
+                logger.error(f"Unknown return code from handle_update_status_request() = {result}", exc_info=True)
                 return Response(f"Unknown return code from handle_update_status_request() = {result}", 500)
 
         except ObjectDoesNotExist:
-            logger.error("The specified app instance was not found release=%s.", release)
-            return Response(f"The specified app instance was not found {release=}.", 404)
+            # This is often not a problem. It typically happens during app re-deployemnts.
+            logger.warning(f"The specified app instance was not found release={release}")
+            return Response(f"The specified app instance was not found {release=}", 404)
 
         except Exception as err:
-            logger.error(
-                "Unable to update the status of the specified app instance %s. %s, %s", release, err, type(err)
-            )
+            logger.error(f"Unable to update the status of the specified app instance {release}. {err}, {type(err)}")
             return Response(f"Unable to update the status of the specified app instance {release=}.", 500)
 
     # GET verb

apps/constants.py

@@ -20,4 +20,6 @@
     "3000-9999.",
     "note_on_linkonly_privacy": "This option can be used only for a limited amount of time, for example while under "
     "development or during peer review.",
+    "environment": "Select the environment that you want to use for your app. The environment is a Docker image that "
+    "contains the software and dependencies needed to run your app.",
 }

apps/forms/__init__.py

@@ -1,4 +1,4 @@
-from .custom_field import CustomField  # isort:skip
+from apps.forms.field.custom import CustomField  # isort:skip
 from .base import AppBaseForm, BaseForm
 from .custom import CustomAppForm
 from .dash import DashForm

apps/forms/base.py

@@ -4,44 +4,15 @@
 from crispy_forms.layout import Button, Div, Submit
 from django import forms
 from django.shortcuts import get_object_or_404
-from django.template import loader
-from django.utils.safestring import mark_safe
 
-from apps.constants import HELP_MESSAGE_MAP
-from apps.forms import CustomField
-from apps.helpers import get_select_options
+from apps.forms.field.widget import SubdomainInputGroup
 from apps.models import BaseAppInstance, Subdomain, VolumeInstance
 from apps.types_.subdomain import SubdomainCandidateName, SubdomainTuple
 from projects.models import Flavor, Project
 
 __all__ = ["BaseForm", "AppBaseForm"]
 
 
-# Custom Widget that adds boostrap-style input group to the subdomain field
-class SubdomainInputGroup(forms.Widget):
-    subdomain_template = "apps/partials/subdomain_input_group.html"
-
-    def __init__(self, base_widget, data, *args, **kwargs):
-        # Initialise widget and get base instance
-        super(SubdomainInputGroup, self).__init__(*args, **kwargs)
-        self.base_widget = base_widget(*args, **kwargs)
-        self.data = data
-
-    def get_context(self, name, value, attrs=None):
-        return {
-            "initial_subdomain": value,
-            "project_pk": self.data["project_pk"],
-            "hidden": self.data["hidden"],
-            "subdomain_list": get_select_options(self.data["project_pk"]),
-        }
-
-    def render(self, name, value, attrs=None, renderer=None):
-        # Render base widget and add bootstrap spans
-        context = self.get_context(name, value, attrs)
-        template = loader.get_template(self.subdomain_template).render(context)
-        return mark_safe(template)
-
-
 class BaseForm(forms.ModelForm):
     """The most generic form for apps running on serve. Current intended use is for VolumesK8S type apps"""
 
@@ -146,32 +117,6 @@ def validate_subdomain(self, subdomain_input):
 
         return SubdomainTuple(subdomain_input, True)
 
-    def get_common_field(self, field_name: str, **kwargs):
-        """
-        This function is very useful because it allows you to create a custom field,
-        that has a question_mark with tooltip next to the label. So "Name (?)" will have a tooltip.
-        The text in the tooltip is defined in HELP_MESSAGE_MAP.
-        The CustomField class just inherits the crispy_forms.layout.Field class and adds the
-        help_message attribute to it. The template then uses it to render the tooltip for all fields
-        using this class.
-        """
-
-        spinner = kwargs.pop("spinner", False)
-
-        template = "apps/custom_field.html"
-        base_args = dict(
-            css_class="form-control form-control-with-spinner" if spinner else "form-control",
-            wrapper_class="mb-3",
-            rows=3,
-            help_message=HELP_MESSAGE_MAP.get(field_name, ""),
-            spinner=spinner,
-        )
-
-        base_args.update(kwargs)
-        field = CustomField(field_name, **base_args)
-        field.set_template(template)
-        return Div(field, css_class="form-input-with-spinner" if spinner else None)
-
     class Meta:
         # Specify model to be used
         model = BaseAppInstance