Merge branch 'main' into no-dh-strip-nondeterminism

minvws · Oct 12, 2024 · f2eb9fa · f2eb9fa
2 parents 4659015 + 08aa9e0
commit f2eb9fa
Show file tree

Hide file tree

Showing 22 changed files with 100 additions and 21 deletions.
diff --git a/.github/workflows/test_debian_packages_on_ubuntu.yml b/.github/workflows/test_debian_packages_on_ubuntu.yml
@@ -213,7 +213,7 @@ jobs:
           sudo sed -i "s/BYTES_PASSWORD=\$/BYTES_PASSWORD=$(sudo grep BYTES_PASSWORD /etc/kat/bytes.conf | awk -F'=' '{ print $2 }')/" /etc/kat/mula.conf
 
       - name: Restart KAT
-        run: sudo systemctl restart kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+        run: sudo systemctl restart kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
       - name: Setup accounts in Rocky
         run: |

diff --git a/boefjes/boefjes/app.py b/boefjes/boefjes/app.py
@@ -90,7 +90,7 @@ def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue):
         except HTTPError:
             # Scheduler is having issues, so make note of it and try again
             logger.exception("Getting the queues from the scheduler failed")
-            time.sleep(10 * self.settings.poll_interval)  # But not immediately
+            time.sleep(self.settings.poll_interval)  # But not immediately
             return
 
         # We do not target a specific queue since we start one runtime for all organisations

diff --git a/docker-compose.release-example.yml b/docker-compose.release-example.yml
@@ -98,6 +98,7 @@ services:
   rocky:
     restart: on-failure
     depends_on:
+      - rocky_worker
       - octopoes_api_worker
       - octopoes_api
       - postgres
@@ -108,12 +109,29 @@ services:
     ports:
       - "127.0.0.1:8000:8000"
     image: "ghcr.io/minvws/nl-kat-rocky:${KAT_VERSION}"
+    command: web
     env_file:
       - .env-prod
       - .env
     environment:
       - DATABASE_MIGRATION=true
 
+  rocky_worker:
+    restart: on-failure
+    depends_on:
+      - octopoes_api_worker
+      - octopoes_api
+      - postgres
+      - boefje
+      - normalizer
+      - katalogus
+      - scheduler
+    image: "ghcr.io/minvws/nl-kat-rocky:${KAT_VERSION}"
+    command: worker
+    env_file:
+      - .env-prod
+      - .env
+
   bytes:
     restart: on-failure
     depends_on:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -164,7 +164,7 @@ services:
       target: dev
       args:
         ENVIRONMENT: dev
-    command: python3 manage.py worker
+    command: worker
     volumes:
       - ./rocky:/app/rocky
       - ./octopoes/octopoes:/app/rocky/octopoes

diff --git a/docs/source/installation_and_deployment/containers.md b/docs/source/installation_and_deployment/containers.md
@@ -100,10 +100,12 @@ By default OpenKAT has an IPv6 subnet configured. This configuration (step 4 and
 
 ## Container commands
 
-We have two container images that are used to run multiple containers. What the container runs is be specified by overriding the CMD of the container.
+We have three container images that are used to run multiple containers. What the container runs is be specified by overriding the CMD of the container.
 
 | Container image | CMD         | Description                                                                       |
 | --------------- | ----------- | --------------------------------------------------------------------------------- |
+| rocky           | web         | Django web application                                                            |
+| rocky           | worker      | Rocky reports runner                                                              |
 | boefjes         | boefje      | Boefjes runtime                                                                   |
 | boefjes         | normalizer  | Normalizers runtime                                                               |
 | boefjes         | katalogus   | Katalogus API                                                                     |

diff --git a/docs/source/installation_and_deployment/debianinstall.rst b/docs/source/installation_and_deployment/debianinstall.rst
@@ -292,7 +292,7 @@ After finishing these steps, you should restart KAT to load the new configuratio
 
 .. code-block:: sh
 
-    sudo systemctl restart kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+    sudo systemctl restart kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 Start KAT on system boot
 ========================
@@ -301,7 +301,7 @@ To start KAT when the system boots, enable all KAT services:
 
 .. code-block:: sh
 
-    sudo systemctl enable kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+    sudo systemctl enable kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 
 Configure reverse proxy
@@ -423,4 +423,4 @@ Restart all processes:
 
 .. code-block:: sh
 
-    sudo systemctl restart kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+    sudo systemctl restart kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
diff --git a/docs/source/installation_and_deployment/debuggingtroubleshooting.rst b/docs/source/installation_and_deployment/debuggingtroubleshooting.rst
@@ -45,6 +45,7 @@ The relevant services for OpenKAT:
 * kat-octopoes.service
 * kat-keiko.service
 * kat-rocky.service
+* kat-rocky-worker.service
 * kat-boefjes.service
 * kat-katalogus.service
 * kat-octopoes-worker.service

diff --git a/docs/source/release_notes/1.18.rst b/docs/source/release_notes/1.18.rst
@@ -0,0 +1,26 @@
+============================================
+OpenKAT 1.18
+============================================
+
+New Features
+============
+
+
+Bug fixes
+=========
+
+
+Upgrading
+=========
+
+Debian packages
+---------------
+
+If you are using the :ref:`Scripts` we provide to install/upgrade OpenKAT you
+need to get the latest version that includes the kat-rocky-worker service.
+
+Full Changelog
+==============
+
+The full changelog can be found on `Github
+<https://github.com/minvws/nl-kat-coordination/releases/tag/v1.18.0>`_.
diff --git a/rocky/debian/kat-rocky-worker.service b/rocky/debian/kat-rocky-worker.service
@@ -0,0 +1,18 @@
+[Unit]
+Description=kat-rocky worker
+After=network.target
+
+[Service]
+User=kat
+Group=kat
+SyslogIdentifier=kat-rocky-worker
+WorkingDirectory=/opt/venvs/kat-rocky/
+EnvironmentFile=/usr/lib/kat/rocky.defaults
+EnvironmentFile=/etc/kat/rocky.conf
+ExecStart=/usr/bin/rocky-cli worker
+Restart=on-failure
+RestartSec=3s
+KillMode=mixed
+
+[Install]
+WantedBy=multi-user.target
diff --git a/rocky/debian/kat-rocky.service b/rocky/debian/kat-rocky.service
@@ -1,5 +1,6 @@
 [Unit]
 Description=kat-rocky daemon
+Requires=kat-rocky-worker.service
 After=network.target
 
 [Service]

diff --git a/rocky/debian/rules b/rocky/debian/rules
@@ -50,6 +50,10 @@ override_dh_virtualenv:
 override_dh_gencontrol:
 	dh_gencontrol -- -Vpython=`py3versions -d`
 
+override_dh_installsystemd:
+	dh_installsystemd --name=kat-rocky
+	dh_installsystemd --name=kat-rocky-worker
+
 execute_after_dh_install:
 	dh_installsysusers
 

diff --git a/rocky/entrypoint.sh b/rocky/entrypoint.sh
@@ -8,4 +8,10 @@ if [ "$DATABASE_MIGRATION" = "1" ] || [[ $DATABASE_MIGRATION == "true" ]]; then
     python manage.py migrate --noinput
 fi
 
+if [ "$1" = "web" ]; then
+    exec granian --interface wsgi rocky.wsgi:application --host 0.0.0.0
+elif [ "$1" = "worker" ]; then
+    exec python3 manage.py worker
+fi
+
 exec "$@"
diff --git a/rocky/reports/runner/worker.py b/rocky/reports/runner/worker.py
@@ -79,7 +79,7 @@ def _fill_queue(self, task_queue: Queue):
         except HTTPError:
             # Scheduler is having issues, so make note of it and try again
             logger.exception("Getting the queues from the scheduler failed")
-            time.sleep(10 * self.poll_interval)  # But not immediately
+            time.sleep(self.poll_interval)  # But not immediately
             return
 
         # We do not target a specific queue since we start one runtime for all organisations

diff --git a/rocky/rocky/views/upload_raw.py b/rocky/rocky/views/upload_raw.py
@@ -62,7 +62,9 @@ def process_raw(self, form):
         raw_file = form.cleaned_data["raw_file"]
         mime_types = form.cleaned_data["mime_types"]
         input_ooi = form.cleaned_data["ooi"]
-        valid_time = form.cleaned_data["date"].replace(tzinfo=timezone.utc)
+        valid_time = None
+        if "date" in form.cleaned_data:
+            valid_time = form.cleaned_data["date"].replace(tzinfo=timezone.utc)
 
         try:
             get_bytes_client(self.organization.code).upload_raw(

diff --git a/rocky/tools/forms/upload_raw.py b/rocky/tools/forms/upload_raw.py
@@ -25,10 +25,11 @@ class UploadRawForm(BaseRockyForm):
         required=True,
         widget=forms.TextInput(attrs={"placeholder": "text/html, image/jpeg, ..."}),
     )
-    raw_file = forms.FileField(label=_("Upload raw file"), allow_empty_file=False)
+    raw_file = forms.FileField(label=_("Upload raw file"), allow_empty_file=False, required=True)
 
     ooi_id = forms.CharField(
         label="Input or Scan OOI",
+        required=True,
         widget=DataListInput(
             attrs={"placeholder": _("Click to select one of the available options, or type one yourself")}
         ),
@@ -64,7 +65,7 @@ def clean(self):
         cleaned_data = super().clean()
 
         date = self.cleaned_data["date"]
-        ooi_id = self.cleaned_data["ooi_id"]
+        ooi_id = self.data["ooi_id"]
 
         # date should not be in the future
         if date > datetime.now(tz=timezone.utc):

diff --git a/scripts/installation/openkat-empty-job-queue.sh b/scripts/installation/openkat-empty-job-queue.sh
@@ -2,7 +2,7 @@
 
 # Stop openKAT
 echo "Stopping openKAT processes"
-sudo systemctl stop xtdb-http-multinode kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl stop xtdb-http-multinode kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 # Start postgres, switch to the mula_db and empty the job queue
 echo "Emptying job queue"
@@ -13,6 +13,6 @@ EOF
 
 # Start openKAT
 echo "Starting openKAT processes"
-sudo systemctl start xtdb-http-multinode kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl start xtdb-http-multinode kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 echo "End of script. It might take a few more seconds for OpenKAT to be fully started and available."
diff --git a/scripts/installation/openkat-install.sh b/scripts/installation/openkat-install.sh
@@ -285,9 +285,9 @@ echo "Step 6.11 - Set kat permissions in rabbitmq"
 sudo rabbitmqctl set_permissions -p "kat" "kat" ".*" ".*" ".*"
 
 echo "Step 7 - Configure start at system boot"
-sudo systemctl enable kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl enable kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 echo "Step 8 - Restart OpenKAT"
-sudo systemctl restart kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl restart kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 echo "Step 9 - End of OpenKAT install script"
diff --git a/scripts/installation/openkat-reset.sh b/scripts/installation/openkat-reset.sh
@@ -29,7 +29,7 @@ fi
 pushd /
 
 echo "Stop OpenKAT"
-sudo systemctl stop xtdb-http-multinode kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl stop xtdb-http-multinode kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 echo "Delete XTDB databases"
 sudo rm -rf /var/lib/xtdb/*
@@ -89,6 +89,6 @@ if [[ ${1} != "no_super_user" ]]; then
 fi
 
 echo "Start OpenKAT"
-sudo systemctl start xtdb-http-multinode kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl start xtdb-http-multinode kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 popd
diff --git a/scripts/installation/openkat-restart.sh b/scripts/installation/openkat-restart.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 echo "Restarting openKAT..."
-sudo systemctl restart xtdb-http-multinode kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl restart xtdb-http-multinode kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
diff --git a/scripts/installation/openkat-start.sh b/scripts/installation/openkat-start.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 echo "Starting openKAT..."
-sudo systemctl start xtdb-http-multinode kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl start xtdb-http-multinode kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
diff --git a/scripts/installation/openkat-stop.sh b/scripts/installation/openkat-stop.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 echo "Stopping openKAT..."
-sudo systemctl stop xtdb-http-multinode kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl stop xtdb-http-multinode kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
diff --git a/scripts/installation/openkat-update.sh b/scripts/installation/openkat-update.sh
@@ -82,6 +82,6 @@ sudo -u kat update-katalogus-db
 sudo -u kat update-mula-db
 
 echo "Step 7 - Restart OpenKAT"
-sudo systemctl restart xtdb-http-multinode kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+sudo systemctl restart xtdb-http-multinode kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
 echo "End of OpenKAT update script"