From 0a364cccb584a7a1b709012fe2fedcabe9d46d1a Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 31 Oct 2024 17:41:06 +0100
Subject: [PATCH 01/22] log-collector DaemonSet + script changes

---
 packages/log-collector.yaml | 39 +++++++++++++++++++++++++++++++++++++
 packages/scripts.nix        | 24 ++++++-----------------
 2 files changed, 45 insertions(+), 18 deletions(-)
 create mode 100644 packages/log-collector.yaml

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
new file mode 100644
index 0000000000..3e40228d61
--- /dev/null
+++ b/packages/log-collector.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: log-collector
+  namespace: @@NAMESPACE@@
+spec:
+  selector:
+    matchLabels:
+      name: log-collector
+  template:
+    metadata:
+      labels:
+        name: log-collector
+    spec:
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: log-collector
+          image: docker.io/bash:latest # TODO(miampf): Replace with non docker.io image
+          volumeMounts:
+            - mountPath: /logs
+              name: log-volume
+              readOnly: true
+          command:
+            - /usr/local/bin/bash
+            - -c
+            - |
+              ls /logs
+      volumes:
+        - name: log-volume
+          # mount the nodes logs to the container
+          hostPath:
+            path: /var/log/pods
+            type: Directory
diff --git a/packages/scripts.nix b/packages/scripts.nix
index a116e9343f..0163275947 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -429,28 +429,16 @@
     name = "get-logs";
     runtimeInputs = with pkgs; [ kubectl ];
     text = ''
-      set -euo pipefail
-      # wait until namespace file is populated
       while ! [[ -s "$1" ]]; do
         sleep 1
       done
       namespace="$(head -n1 "$1")"
-      while kubectl get ns "$namespace" 1>/dev/null 2>/dev/null; do
-        pods="$(kubectl get pods -n "$namespace" | awk '!/^NAME/{print $1}')"
-        mkdir -p "workspace/namespace-logs"
-        for pod in $pods; do
-          logfile="workspace/namespace-logs/$pod.log"
-          if ! [[ -f "$logfile" ]]; then
-            {
-              touch "$logfile" # prevents creation of to much processes
-              # wait for all containers of the pod to come online, then collect the logs
-              kubectl wait pod --all --for=condition=Ready --timeout="-1s" -n "$namespace" "$pod" 1>/dev/null 2>/dev/null
-              kubectl logs -f --all-containers=true -n "$namespace" "$pod" > "$logfile"
-            } &
-          fi
-        done
-      done
-      wait
+
+      cp ./packages/log-collector.yaml ./workspace/log-collector.yaml
+      sed -i "s/@@NAMESPACE@@/''${namespace}/g" ./workspace/log-collector.yaml
+
+      kubectl get namespace | grep -q "^$namespace" || kubectl create namespace "$namespace"
+      kubectl apply -f ./workspace/log-collector.yaml
     '';
   };
 

From 2b11c2bdbc5211d8495732ebd601540b9bfb5e59 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 5 Nov 2024 14:21:47 +0100
Subject: [PATCH 02/22] don't create namespace yourself

---
 packages/scripts.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/scripts.nix b/packages/scripts.nix
index 0163275947..d6aa940230 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -437,7 +437,6 @@
       cp ./packages/log-collector.yaml ./workspace/log-collector.yaml
       sed -i "s/@@NAMESPACE@@/''${namespace}/g" ./workspace/log-collector.yaml
 
-      kubectl get namespace | grep -q "^$namespace" || kubectl create namespace "$namespace"
       kubectl apply -f ./workspace/log-collector.yaml
     '';
   };

From 1d569fc5bfa85d4676993306ddb89d4b9601df72 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 5 Nov 2024 14:21:55 +0100
Subject: [PATCH 03/22] use inotifywait

---
 packages/log-collector.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index 3e40228d61..332cdbf57e 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -30,7 +30,11 @@ spec:
             - /usr/local/bin/bash
             - -c
             - |
-              ls /logs
+              inotifywait -m /logs -e create -e moved_to |
+                while read path action file; do
+                  echo "$file was created."
+                  tail --follow=name -v "$file" &
+                done
       volumes:
         - name: log-volume
           # mount the nodes logs to the container

From 35b35b51cd7c58623bfb583fe97eca38ec5dc353 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 5 Nov 2024 15:36:08 +0100
Subject: [PATCH 04/22] correct pathing & temporarily use ubuntu image

---
 packages/log-collector.yaml | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index 332cdbf57e..0be3bcbc4e 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -21,19 +21,26 @@ spec:
           effect: NoSchedule
       containers:
         - name: log-collector
-          image: docker.io/bash:latest # TODO(miampf): Replace with non docker.io image
+          image: docker.io/ubuntu:oracular # TODO(miampf): Replace with non docker.io image
           volumeMounts:
             - mountPath: /logs
               name: log-volume
               readOnly: true
           command:
-            - /usr/local/bin/bash
+            - /usr/bin/bash
             - -c
             - |
-              inotifywait -m /logs -e create -e moved_to |
+              # TODO(miampf): Prepare an image that already has inotify-tools installed
+              apt-get update && apt-get install -y inotify-tools
+              inotifywait -m /logs -r -e create -e moved_to |
                 while read path action file; do
-                  echo "$file was created."
-                  tail --follow=name -v "$file" &
+                  echo "DEBUG: $path; $action; $file"
+                  filepath="$path$file"
+                  echo "DEBUG: $filepath"
+                  if [[ -f "$filepath" ]]; then
+                    echo "$filepath was created."
+                    tail --follow=name -v "$filepath" &
+                  fi
                 done
       volumes:
         - name: log-volume

From e0c7dbef8ada1935680e352dea27255de0ba3d19 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 5 Nov 2024 16:14:47 +0100
Subject: [PATCH 05/22] working compression of logs

---
 packages/log-collector.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index 0be3bcbc4e..f79670f41b 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -30,8 +30,9 @@ spec:
             - /usr/bin/bash
             - -c
             - |
-              # TODO(miampf): Prepare an image that already has inotify-tools installed
-              apt-get update && apt-get install -y inotify-tools
+              # TODO(miampf): Prepare an image that already has required tools installed
+              apt-get update && apt-get install -y inotify-tools ncompress
+              mkdir /export
               inotifywait -m /logs -r -e create -e moved_to |
                 while read path action file; do
                   echo "DEBUG: $path; $action; $file"
@@ -39,7 +40,8 @@ spec:
                   echo "DEBUG: $filepath"
                   if [[ -f "$filepath" ]]; then
                     echo "$filepath was created."
-                    tail --follow=name -v "$filepath" &
+                    mkdir -p "/export$path"
+                    tail --follow=name -v "$filepath" | compress > "/export$filepath.Z" &
                   fi
                 done
       volumes:

From 691ffec1ec19d9156865f0202ce1fac1fa23fd9b Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 5 Nov 2024 16:15:29 +0100
Subject: [PATCH 06/22] remove unnecessary echos

---
 packages/log-collector.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index f79670f41b..e6343f7343 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -35,11 +35,8 @@ spec:
               mkdir /export
               inotifywait -m /logs -r -e create -e moved_to |
                 while read path action file; do
-                  echo "DEBUG: $path; $action; $file"
                   filepath="$path$file"
-                  echo "DEBUG: $filepath"
                   if [[ -f "$filepath" ]]; then
-                    echo "$filepath was created."
                     mkdir -p "/export$path"
                     tail --follow=name -v "$filepath" | compress > "/export$filepath.Z" &
                   fi

From d26bf97de70087fc85efc3b03948168fecf2f26b Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 5 Nov 2024 17:59:29 +0100
Subject: [PATCH 07/22] decompression

---
 packages/log-collector.yaml |  4 ++--
 packages/scripts.nix        | 23 ++++++++++++++++++++++-
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index e6343f7343..74de563231 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -30,15 +30,15 @@ spec:
             - /usr/bin/bash
             - -c
             - |
+              mkdir /export
               # TODO(miampf): Prepare an image that already has required tools installed
               apt-get update && apt-get install -y inotify-tools ncompress
-              mkdir /export
               inotifywait -m /logs -r -e create -e moved_to |
                 while read path action file; do
                   filepath="$path$file"
                   if [[ -f "$filepath" ]]; then
                     mkdir -p "/export$path"
-                    tail --follow=name -v "$filepath" | compress > "/export$filepath.Z" &
+                    tail --follow=name -v "$filepath" | compress -f >"/export$filepath" &
                   fi
                 done
       volumes:
diff --git a/packages/scripts.nix b/packages/scripts.nix
index d6aa940230..35a3f1dac0 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -427,7 +427,10 @@
   # Usage: get-logs $namespaceFile
   get-logs = writeShellApplication {
     name = "get-logs";
-    runtimeInputs = with pkgs; [ kubectl ];
+    runtimeInputs = with pkgs; [ 
+      kubectl
+      ncompress
+    ];
     text = ''
       while ! [[ -s "$1" ]]; do
         sleep 1
@@ -438,6 +441,24 @@
       sed -i "s/@@NAMESPACE@@/''${namespace}/g" ./workspace/log-collector.yaml
 
       kubectl apply -f ./workspace/log-collector.yaml
+
+      pod="$(kubectl get pods -o name -n "$namespace" | grep log-collector | cut -c 5-)"
+      echo "$pod"
+      mkdir -p ./workspace/logs
+      kubectl wait --for=condition=Ready -n "$namespace" "pod/$pod" #1>/dev/null 2>/dev/null
+      echo "DEBUG: Pod $pod running"
+      # Download and extract the logs every 3 seconds
+      while true; do
+        kubectl exec -n "$namespace" "$pod" -- bash -c "rm -f /exported-logs.tar.gz; tar zcvf /exported-logs.tar.gz /export" #1>/dev/null 2>/dev/null
+        echo "DEBUG: compressed archive"
+        rm -f ./workspace/logs/exported-logs.tar.gz
+        kubectl cp -n "$namespace" "$pod:/exported-logs.tar.gz" ./workspace/logs/exported-logs.tar.gz #1>/dev/null 2>/dev/null
+        echo "DEBUG: downloaded archive"
+        tar xzvf ./workspace/logs/exported-logs.tar.gz --directory ./workspace/logs #1>/dev/null 2>/dev/null
+        find ./workspace/logs -type f -exec bash -c compress -d {} \;
+        echo "DEBUG: extracted archive"
+        sleep 3
+      done
     '';
   };
 

From 0f1bae1a0b47442515b85ba5ba4316699d0b42e1 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 12:53:39 +0100
Subject: [PATCH 08/22] dont compress logs on machine

---
 packages/log-collector.yaml | 4 ++--
 packages/scripts.nix        | 2 --
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index 74de563231..c09d4aca81 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -32,13 +32,13 @@ spec:
             - |
               mkdir /export
               # TODO(miampf): Prepare an image that already has required tools installed
-              apt-get update && apt-get install -y inotify-tools ncompress
+              apt-get update && apt-get install -y inotify-tools
               inotifywait -m /logs -r -e create -e moved_to |
                 while read path action file; do
                   filepath="$path$file"
                   if [[ -f "$filepath" ]]; then
                     mkdir -p "/export$path"
-                    tail --follow=name -v "$filepath" | compress -f >"/export$filepath" &
+                    tail --follow=name "$filepath" >"/export$filepath" &
                   fi
                 done
       volumes:
diff --git a/packages/scripts.nix b/packages/scripts.nix
index 35a3f1dac0..b44514fee3 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -429,7 +429,6 @@
     name = "get-logs";
     runtimeInputs = with pkgs; [ 
       kubectl
-      ncompress
     ];
     text = ''
       while ! [[ -s "$1" ]]; do
@@ -455,7 +454,6 @@
         kubectl cp -n "$namespace" "$pod:/exported-logs.tar.gz" ./workspace/logs/exported-logs.tar.gz #1>/dev/null 2>/dev/null
         echo "DEBUG: downloaded archive"
         tar xzvf ./workspace/logs/exported-logs.tar.gz --directory ./workspace/logs #1>/dev/null 2>/dev/null
-        find ./workspace/logs -type f -exec bash -c compress -d {} \;
         echo "DEBUG: extracted archive"
         sleep 3
       done

From 36961c939931c70283af7a2fda2665c4a61258b4 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 12:58:45 +0100
Subject: [PATCH 09/22] prevent printing

---
 packages/scripts.nix | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/packages/scripts.nix b/packages/scripts.nix
index b44514fee3..93b92e9437 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -439,22 +439,18 @@
       cp ./packages/log-collector.yaml ./workspace/log-collector.yaml
       sed -i "s/@@NAMESPACE@@/''${namespace}/g" ./workspace/log-collector.yaml
 
-      kubectl apply -f ./workspace/log-collector.yaml
+      kubectl apply -f ./workspace/log-collector.yaml 1>/dev/null 2>/dev/null
 
       pod="$(kubectl get pods -o name -n "$namespace" | grep log-collector | cut -c 5-)"
       echo "$pod"
       mkdir -p ./workspace/logs
-      kubectl wait --for=condition=Ready -n "$namespace" "pod/$pod" #1>/dev/null 2>/dev/null
-      echo "DEBUG: Pod $pod running"
+      kubectl wait --for=condition=Ready -n "$namespace" "pod/$pod" 1>/dev/null 2>/dev/null
       # Download and extract the logs every 3 seconds
       while true; do
-        kubectl exec -n "$namespace" "$pod" -- bash -c "rm -f /exported-logs.tar.gz; tar zcvf /exported-logs.tar.gz /export" #1>/dev/null 2>/dev/null
-        echo "DEBUG: compressed archive"
+        kubectl exec -n "$namespace" "$pod" -- bash -c "rm -f /exported-logs.tar.gz; tar zcvf /exported-logs.tar.gz /export" 1>/dev/null 2>/dev/null
         rm -f ./workspace/logs/exported-logs.tar.gz
-        kubectl cp -n "$namespace" "$pod:/exported-logs.tar.gz" ./workspace/logs/exported-logs.tar.gz #1>/dev/null 2>/dev/null
-        echo "DEBUG: downloaded archive"
-        tar xzvf ./workspace/logs/exported-logs.tar.gz --directory ./workspace/logs #1>/dev/null 2>/dev/null
-        echo "DEBUG: extracted archive"
+        kubectl cp -n "$namespace" "$pod:/exported-logs.tar.gz" ./workspace/logs/exported-logs.tar.gz 1>/dev/null 2>/dev/null
+        tar xzvf ./workspace/logs/exported-logs.tar.gz --directory ./workspace/logs 1>/dev/null 2>/dev/null
         sleep 3
       done
     '';

From fb00720f23bde27935a44f06448957510e4dfdce Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 12:58:59 +0100
Subject: [PATCH 10/22] kill log collection after e2e test is done

---
 .github/workflows/e2e.yml             | 1 +
 .github/workflows/e2e_aks_runtime.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 1e3ed5d190..8f3af9606f 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -79,6 +79,7 @@ jobs:
             --namespace-file workspace/e2e.namespace \
             --platform ${{ inputs.platform }} \
             --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}"
+          kill %%
       - name: Upload logs
         if: always()
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
diff --git a/.github/workflows/e2e_aks_runtime.yml b/.github/workflows/e2e_aks_runtime.yml
index 064e4a3cbc..5a1b7bee62 100644
--- a/.github/workflows/e2e_aks_runtime.yml
+++ b/.github/workflows/e2e_aks_runtime.yml
@@ -87,6 +87,7 @@ jobs:
             --namespace-file workspace/e2e.namespace \
             --platform AKS-CLH-SNP \
             --skip-undeploy="false"
+          kill %%
       - name: Upload logs
         if: always()
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3

From d0c8714fbac32b9828e1509fe8c94544a842b62e Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 12:59:31 +0100
Subject: [PATCH 11/22] pipefail

---
 packages/scripts.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/scripts.nix b/packages/scripts.nix
index 93b92e9437..a71df42db1 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -431,6 +431,7 @@
       kubectl
     ];
     text = ''
+      set -euo pipefail
       while ! [[ -s "$1" ]]; do
         sleep 1
       done

From 450cc943306d72a5804d3cdb144dceb8a3d58aca Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 13:05:28 +0100
Subject: [PATCH 12/22] upload the new log path

---
 .github/workflows/e2e.yml             | 2 +-
 .github/workflows/e2e_aks_runtime.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 8f3af9606f..50b12315c2 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -85,7 +85,7 @@ jobs:
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: e2e_pod_logs-${{ inputs.platform }}-${{ inputs.test-name }}
-          path: workspace/namespace-logs
+          path: workspace/logs/export/logs
       - name: Notify teams channel of failure
         if: ${{ failure() && github.event_name == 'schedule' && github.run_attempt == 1 }}
         uses: ./.github/actions/post_to_teams
diff --git a/.github/workflows/e2e_aks_runtime.yml b/.github/workflows/e2e_aks_runtime.yml
index 5a1b7bee62..538302cffe 100644
--- a/.github/workflows/e2e_aks_runtime.yml
+++ b/.github/workflows/e2e_aks_runtime.yml
@@ -93,7 +93,7 @@ jobs:
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: e2e_pod_logs-AKS-CLH-SNP-aks-runtime
-          path: workspace/namespace-logs
+          path: workspace/logs/export/logs
       - name: Notify teams channel of failure
         if: ${{ failure() && github.event_name == 'schedule' && github.run_attempt == 1 }}
         uses: ./.github/actions/post_to_teams

From e3449ec86138f760fa98d420c464c53471a8fa84 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 13:22:22 +0100
Subject: [PATCH 13/22] add priority class to prioritize log collector

---
 packages/log-collector.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index c09d4aca81..bd029e39d8 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -12,6 +12,7 @@ spec:
       labels:
         name: log-collector
     spec:
+      priorityClassName: high-priority-logcollector
       tolerations:
         - key: node-role.kubernetes.io/control-plane
           operator: Exists
@@ -47,3 +48,11 @@ spec:
           hostPath:
             path: /var/log/pods
             type: Directory
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: high-priority-logcollector
+value: 10000000
+globalDefault: false
+description: "This priority class is used to prioritise the log collector pod creation before anything else"

From 467ed32c034c1e1d2c02f4c13e2b626cb0f2c170 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 14:14:44 +0100
Subject: [PATCH 14/22] collect missed logs + only collect from own ns

---
 packages/log-collector.yaml | 14 +++++++++++++-
 packages/scripts.nix        |  1 -
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index bd029e39d8..a0adb856f6 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -27,6 +27,11 @@ spec:
             - mountPath: /logs
               name: log-volume
               readOnly: true
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
           command:
             - /usr/bin/bash
             - -c
@@ -34,10 +39,17 @@ spec:
               mkdir /export
               # TODO(miampf): Prepare an image that already has required tools installed
               apt-get update && apt-get install -y inotify-tools
+              # collect all logs that may have been missed during startup
+              for file in $(find /logs -name *.log); do
+                if [[ -f "$file" && "$file" == *"$POD_NAMESPACE"* ]]; then
+                  mkdir -p "/export$(dirname "$file")"
+                  cp "$file" "/export$file"
+                fi
+              done
               inotifywait -m /logs -r -e create -e moved_to |
                 while read path action file; do
                   filepath="$path$file"
-                  if [[ -f "$filepath" ]]; then
+                  if [[ -f "$filepath" && "$filepath" == *"$POD_NAMESPACE"* ]]; then
                     mkdir -p "/export$path"
                     tail --follow=name "$filepath" >"/export$filepath" &
                   fi
diff --git a/packages/scripts.nix b/packages/scripts.nix
index a71df42db1..63c1b699b3 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -443,7 +443,6 @@
       kubectl apply -f ./workspace/log-collector.yaml 1>/dev/null 2>/dev/null
 
       pod="$(kubectl get pods -o name -n "$namespace" | grep log-collector | cut -c 5-)"
-      echo "$pod"
       mkdir -p ./workspace/logs
       kubectl wait --for=condition=Ready -n "$namespace" "pod/$pod" 1>/dev/null 2>/dev/null
       # Download and extract the logs every 3 seconds

From b2f9282e64ffd8fd82b50ad8f05353344d6c5819 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 16:12:19 +0100
Subject: [PATCH 15/22] define docker image

---
 .../by-name/k8s-log-collector/package.nix     | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 packages/by-name/k8s-log-collector/package.nix

diff --git a/packages/by-name/k8s-log-collector/package.nix b/packages/by-name/k8s-log-collector/package.nix
new file mode 100644
index 0000000000..b2713c5e08
--- /dev/null
+++ b/packages/by-name/k8s-log-collector/package.nix
@@ -0,0 +1,47 @@
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+{
+  dockerTools,
+  writeShellApplication,
+  inotify-tools,
+  busybox
+}:
+
+let
+  collection-script = writeShellApplication {
+    name = "collect-logs";
+    runtimeInputs = [
+      inotify-tools
+      busybox
+    ];
+    text = ''
+      set -euo pipefail
+      mkdir /export
+      # collect all logs that may have been missed during startup
+      find /logs -name "*.log" |
+        while read -r file; do
+          if [[ -f "$file" && "$file" == *"$POD_NAMESPACE"* ]]; then
+            mkdir -p "/export$(dirname "$file")"
+            tail --follow=name "$file" > "export$file" &
+          fi
+        done
+      inotifywait -m /logs -r -e create -e moved_to |
+        while read -r path _action file; do
+          filepath="$path$file"
+          if [[ -f "$filepath" && "$filepath" == *"$POD_NAMESPACE"* ]]; then
+            mkdir -p "/export$path"
+            tail --follow=name "$filepath" >"/export$filepath" &
+          fi
+        done
+    '';
+  };
+in
+dockerTools.buildLayeredImage {
+  name = "k8s-log-collector";
+  tag = "0.1.0";
+  config = {
+    Cmd = [ "${collection-script}/bin/collect-logs" ];
+    Volumes = { "/logs" = {}; };
+  };
+}

From 3a2d351c557999f75bd53796ca1fd6787d8a4f4b Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 16:12:30 +0100
Subject: [PATCH 16/22] add license info

---
 packages/log-collector.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index a0adb856f6..548cb4b7a2 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -1,3 +1,6 @@
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:

From 38ab6bba21480c7a74804f888761afa68bea4b74 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 7 Nov 2024 16:51:06 +0100
Subject: [PATCH 17/22] utilise new image + adjust script

---
 .../by-name/k8s-log-collector/package.nix     | 24 +++++++++++++++----
 packages/log-collector.yaml                   | 24 +------------------
 packages/scripts.nix                          |  2 +-
 3 files changed, 22 insertions(+), 28 deletions(-)

diff --git a/packages/by-name/k8s-log-collector/package.nix b/packages/by-name/k8s-log-collector/package.nix
index b2713c5e08..5a81ccc850 100644
--- a/packages/by-name/k8s-log-collector/package.nix
+++ b/packages/by-name/k8s-log-collector/package.nix
@@ -4,8 +4,13 @@
 {
   dockerTools,
   writeShellApplication,
+  buildEnv,
   inotify-tools,
-  busybox
+  coreutils,
+  findutils,
+  bash,
+  gnutar,
+  gzip
 }:
 
 let
@@ -13,7 +18,8 @@ let
     name = "collect-logs";
     runtimeInputs = [
       inotify-tools
-      busybox
+      coreutils
+      findutils
     ];
     text = ''
       set -euo pipefail
@@ -23,7 +29,7 @@ let
         while read -r file; do
           if [[ -f "$file" && "$file" == *"$POD_NAMESPACE"* ]]; then
             mkdir -p "/export$(dirname "$file")"
-            tail --follow=name "$file" > "export$file" &
+            tail --follow=name "$file" >"/export$file" &
           fi
         done
       inotifywait -m /logs -r -e create -e moved_to |
@@ -37,9 +43,19 @@ let
     '';
   };
 in
-dockerTools.buildLayeredImage {
+dockerTools.buildImage {
   name = "k8s-log-collector";
   tag = "0.1.0";
+  copyToRoot = buildEnv {
+    name = "bin";
+    paths = [ 
+      bash
+      coreutils
+      gnutar
+      gzip
+    ];
+    pathsToLink = "/bin";
+  };
   config = {
     Cmd = [ "${collection-script}/bin/collect-logs" ];
     Volumes = { "/logs" = {}; };
diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index 548cb4b7a2..01845eb264 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -25,7 +25,7 @@ spec:
           effect: NoSchedule
       containers:
         - name: log-collector
-          image: docker.io/ubuntu:oracular # TODO(miampf): Replace with non docker.io image
+          image: "ghcr.io/miampf/k8s-log-collector:latest"
           volumeMounts:
             - mountPath: /logs
               name: log-volume
@@ -35,28 +35,6 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-          command:
-            - /usr/bin/bash
-            - -c
-            - |
-              mkdir /export
-              # TODO(miampf): Prepare an image that already has required tools installed
-              apt-get update && apt-get install -y inotify-tools
-              # collect all logs that may have been missed during startup
-              for file in $(find /logs -name *.log); do
-                if [[ -f "$file" && "$file" == *"$POD_NAMESPACE"* ]]; then
-                  mkdir -p "/export$(dirname "$file")"
-                  cp "$file" "/export$file"
-                fi
-              done
-              inotifywait -m /logs -r -e create -e moved_to |
-                while read path action file; do
-                  filepath="$path$file"
-                  if [[ -f "$filepath" && "$filepath" == *"$POD_NAMESPACE"* ]]; then
-                    mkdir -p "/export$path"
-                    tail --follow=name "$filepath" >"/export$filepath" &
-                  fi
-                done
       volumes:
         - name: log-volume
           # mount the nodes logs to the container
diff --git a/packages/scripts.nix b/packages/scripts.nix
index 63c1b699b3..90135f3941 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -447,7 +447,7 @@
       kubectl wait --for=condition=Ready -n "$namespace" "pod/$pod" 1>/dev/null 2>/dev/null
       # Download and extract the logs every 3 seconds
       while true; do
-        kubectl exec -n "$namespace" "$pod" -- bash -c "rm -f /exported-logs.tar.gz; tar zcvf /exported-logs.tar.gz /export" 1>/dev/null 2>/dev/null
+        kubectl exec -n "$namespace" "$pod" -- /bin/bash -c "rm -f /exported-logs.tar.gz; tar zcvf /exported-logs.tar.gz /export" 1>/dev/null 2>/dev/null
         rm -f ./workspace/logs/exported-logs.tar.gz
         kubectl cp -n "$namespace" "$pod:/exported-logs.tar.gz" ./workspace/logs/exported-logs.tar.gz 1>/dev/null 2>/dev/null
         tar xzvf ./workspace/logs/exported-logs.tar.gz --directory ./workspace/logs 1>/dev/null 2>/dev/null

From b585c97019eb3b06ac662cbb0f6d0cf4843ff8e0 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 12 Nov 2024 15:26:10 +0100
Subject: [PATCH 18/22] use pinned edgelesssys ghcr image

---
 packages/log-collector.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index 01845eb264..f4f88e1e3d 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -25,7 +25,7 @@ spec:
           effect: NoSchedule
       containers:
         - name: log-collector
-          image: "ghcr.io/miampf/k8s-log-collector:latest"
+          image: "ghcr.io/edgelesssys/k8s-log-collector@sha256:fd173230870b9e19a342627e31a50a0d6e45e7c8770c133b62e72cb4e898bc3e"
           volumeMounts:
             - mountPath: /logs
               name: log-volume

From 8f58c25480a07c8b27ff2c5ae7c1769dc81a3aa0 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 12 Nov 2024 15:27:40 +0100
Subject: [PATCH 19/22] fmt

---
 packages/by-name/k8s-log-collector/package.nix | 8 +++++---
 packages/log-collector.yaml                    | 2 +-
 packages/scripts.nix                           | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/packages/by-name/k8s-log-collector/package.nix b/packages/by-name/k8s-log-collector/package.nix
index 5a81ccc850..ecd08e6f0b 100644
--- a/packages/by-name/k8s-log-collector/package.nix
+++ b/packages/by-name/k8s-log-collector/package.nix
@@ -10,7 +10,7 @@
   findutils,
   bash,
   gnutar,
-  gzip
+  gzip,
 }:
 
 let
@@ -48,7 +48,7 @@ dockerTools.buildImage {
   tag = "0.1.0";
   copyToRoot = buildEnv {
     name = "bin";
-    paths = [ 
+    paths = [
       bash
       coreutils
       gnutar
@@ -58,6 +58,8 @@ dockerTools.buildImage {
   };
   config = {
     Cmd = [ "${collection-script}/bin/collect-logs" ];
-    Volumes = { "/logs" = {}; };
+    Volumes = {
+      "/logs" = { };
+    };
   };
 }
diff --git a/packages/log-collector.yaml b/packages/log-collector.yaml
index f4f88e1e3d..ab31e585d8 100644
--- a/packages/log-collector.yaml
+++ b/packages/log-collector.yaml
@@ -5,7 +5,7 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: log-collector
-  namespace: @@NAMESPACE@@
+  namespace: "@@NAMESPACE@@"
 spec:
   selector:
     matchLabels:
diff --git a/packages/scripts.nix b/packages/scripts.nix
index 90135f3941..5d55941f60 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -427,7 +427,7 @@
   # Usage: get-logs $namespaceFile
   get-logs = writeShellApplication {
     name = "get-logs";
-    runtimeInputs = with pkgs; [ 
+    runtimeInputs = with pkgs; [
       kubectl
     ];
     text = ''

From 170e66ad125f9596703d77a1f8d6ab31bc463e03 Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 19 Nov 2024 12:49:09 +0100
Subject: [PATCH 20/22] adjust get-logs script

---
 packages/scripts.nix | 54 +++++++++++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/packages/scripts.nix b/packages/scripts.nix
index 5d55941f60..728c92e3e5 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -424,7 +424,7 @@
     '';
   };
 
-  # Usage: get-logs $namespaceFile
+  # Usage: get-logs [start | download] $namespaceFile
   get-logs = writeShellApplication {
     name = "get-logs";
     runtimeInputs = with pkgs; [
@@ -432,27 +432,45 @@
     ];
     text = ''
       set -euo pipefail
-      while ! [[ -s "$1" ]]; do
-        sleep 1
-      done
-      namespace="$(head -n1 "$1")"
-
-      cp ./packages/log-collector.yaml ./workspace/log-collector.yaml
-      sed -i "s/@@NAMESPACE@@/''${namespace}/g" ./workspace/log-collector.yaml
-
-      kubectl apply -f ./workspace/log-collector.yaml 1>/dev/null 2>/dev/null
 
-      pod="$(kubectl get pods -o name -n "$namespace" | grep log-collector | cut -c 5-)"
-      mkdir -p ./workspace/logs
-      kubectl wait --for=condition=Ready -n "$namespace" "pod/$pod" 1>/dev/null 2>/dev/null
-      # Download and extract the logs every 3 seconds
-      while true; do
+      if [[ -z "''${1-}" ]]; then
+        echo "Usage: get-logs [start | download] namespaceFile"
+        exit 1
+      fi
+      case $1 in
+      start)
+        if [[ -z "''${2-}" ]]; then
+          echo "Please add the path to the namespace file."
+          echo "Usage: get-logs start namespaceFile"
+          exit 1
+        fi
+        while ! [[ -s "$2" ]]; do
+          sleep 1
+        done
+        namespace="$(head -n1 "$2")"
+        cp ./packages/log-collector.yaml ./workspace/log-collector.yaml
+        sed -i "s/@@NAMESPACE@@/''${namespace}/g" ./workspace/log-collector.yaml
+        kubectl apply -f ./workspace/log-collector.yaml 1>/dev/null 2>/dev/null
+        ;;
+      download)
+        if [[ -z "''${2-}" ]]; then
+          echo "Please add the path to the namespace file."
+          echo "Usage: get-logs download namespaceFile"
+          exit 1
+        fi
+        namespace="$(head -n1 "$2")"
+        pod="$(kubectl get pods -o name -n "$namespace" | grep log-collector | cut -c 5-)"
+        mkdir -p ./workspace/logs
+        kubectl wait --for=condition=Ready -n "$namespace" "pod/$pod" 1>/dev/null 2>/dev/null
         kubectl exec -n "$namespace" "$pod" -- /bin/bash -c "rm -f /exported-logs.tar.gz; tar zcvf /exported-logs.tar.gz /export" 1>/dev/null 2>/dev/null
-        rm -f ./workspace/logs/exported-logs.tar.gz
         kubectl cp -n "$namespace" "$pod:/exported-logs.tar.gz" ./workspace/logs/exported-logs.tar.gz 1>/dev/null 2>/dev/null
         tar xzvf ./workspace/logs/exported-logs.tar.gz --directory ./workspace/logs 1>/dev/null 2>/dev/null
-        sleep 3
-      done
+        ;;
+      *)
+        echo "Unknown option $1"
+        echo "Usage: get-logs [start | download] namespaceFile"
+        exit 1
+      esac
     '';
   };
 

From cd87fca5880ad294e4081d7bb6c32b242e45da3a Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 19 Nov 2024 12:50:57 +0100
Subject: [PATCH 21/22] adjust workflows to new script

---
 .github/workflows/e2e.yml             | 4 ++--
 .github/workflows/e2e_aks_runtime.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 50b12315c2..55be78ccb1 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -73,13 +73,13 @@ jobs:
           just coordinator initializer port-forwarder openssl cryptsetup service-mesh-proxy node-installer ${{ inputs.platform }}
       - name: E2E Test
         run: |
-          nix run .#scripts.get-logs workspace/e2e.namespace &
+          nix run .#scripts.get-logs start workspace/e2e.namespace &
           nix shell -L .#contrast.e2e --command ${{ inputs.test-name }}.test -test.v \
             --image-replacements workspace/just.containerlookup \
             --namespace-file workspace/e2e.namespace \
             --platform ${{ inputs.platform }} \
             --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}"
-          kill %%
+          nix run .#scripts.get-logs download workspace/e2e.namespace
       - name: Upload logs
         if: always()
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
diff --git a/.github/workflows/e2e_aks_runtime.yml b/.github/workflows/e2e_aks_runtime.yml
index 538302cffe..23e5ddfa7f 100644
--- a/.github/workflows/e2e_aks_runtime.yml
+++ b/.github/workflows/e2e_aks_runtime.yml
@@ -80,14 +80,14 @@ jobs:
           az extension add --name confcom
       - name: E2E test
         run: |
-          nix run .#scripts.get-logs workspace/e2e.namespace &
+          nix run .#scripts.get-logs start workspace/e2e.namespace &
           nix build .#contrast.e2e
           ./result/bin/aks-runtime.test -test.v \
             --image-replacements workspace/just.containerlookup \
             --namespace-file workspace/e2e.namespace \
             --platform AKS-CLH-SNP \
             --skip-undeploy="false"
-          kill %%
+          nix run .#scripts.get-logs download workspace/e2e.namespace
       - name: Upload logs
         if: always()
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3

From fe1570d264dbee268e09fce4f769dc350aacc76b Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Thu, 21 Nov 2024 11:38:26 +0100
Subject: [PATCH 22/22] better argument checking

---
 packages/scripts.nix | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/packages/scripts.nix b/packages/scripts.nix
index 728c92e3e5..3f977ec2af 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -433,17 +433,12 @@
     text = ''
       set -euo pipefail
 
-      if [[ -z "''${1-}" ]]; then
+      if [[ $# -lt 2 ]]; then
         echo "Usage: get-logs [start | download] namespaceFile"
         exit 1
       fi
       case $1 in
       start)
-        if [[ -z "''${2-}" ]]; then
-          echo "Please add the path to the namespace file."
-          echo "Usage: get-logs start namespaceFile"
-          exit 1
-        fi
         while ! [[ -s "$2" ]]; do
           sleep 1
         done
@@ -453,11 +448,6 @@
         kubectl apply -f ./workspace/log-collector.yaml 1>/dev/null 2>/dev/null
         ;;
       download)
-        if [[ -z "''${2-}" ]]; then
-          echo "Please add the path to the namespace file."
-          echo "Usage: get-logs download namespaceFile"
-          exit 1
-        fi
         namespace="$(head -n1 "$2")"
         pod="$(kubectl get pods -o name -n "$namespace" | grep log-collector | cut -c 5-)"
         mkdir -p ./workspace/logs