tasks: Regularly clean up leftover containers and outdated images

Our bots accumulate outdated task containers which eat a lot of space
over time. This will become worse now with automated tasks updates from
commit 4824188.

Our machinery also sometimes leaks exited/failed containers, clean them
up as well.

ansible/roles/tasks-systemd/cockpituous-janitor.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Clean up cockpituous cruft
+
+[Service]
+Type=oneshot
+# remove leftover containers
+ExecStart=/bin/sh -ec 'podman ps -q --filter status=exited | xargs -r podman rm'
+# remote old task images; this is opportunistic -- it won't remove used images, but will fail on them
+ExecStart=-/usr/bin/podman rmi --all

ansible/roles/tasks-systemd/cockpituous-janitor.timer

@@ -0,0 +1,7 @@
+[Unit]
+Description=Regularly clean up cockpituous cruft
+
+[Timer]
+OnBootSec=1h
+OnUnitActiveSec=24h
+AccuracySec=15min

ansible/roles/tasks-systemd/tasks/main.yml

@@ -145,6 +145,18 @@
           '--env=COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/github-token',
       ]
 
+- name: Create janitor service
+  copy:
+    src: "{{ role_path }}/cockpituous-janitor.service"
+    dest: /etc/systemd/system/cockpituous-janitor.service
+    mode: preserve
+
+- name: Create janitor timer
+  copy:
+    src: "{{ role_path }}/cockpituous-janitor.timer"
+    dest: /etc/systemd/system/cockpituous-janitor.timer
+    mode: preserve
+
 - name: Set up systemd service for cockpit/tasks
   shell: |
     export INSTANCES={{ instances | default(1) }}

tasks/install-service

@@ -35,6 +35,7 @@ cat <<EOF > /etc/systemd/system/[email protected]
 Description=Cockpit Tasks %i
 Requires=podman.socket
 After=podman.socket
+Wants=cockpituous-janitor.timer
 
 [Service]
 Restart=always