e2e: upload container logs as artifact (#825)

edgelesssys · Aug 29, 2024 · 548e035 · 548e035
1 parent 7828345
commit 548e035
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 0 deletions.
diff --git a/.github/workflows/e2e_kubernetes.yaml b/.github/workflows/e2e_kubernetes.yaml
@@ -69,11 +69,18 @@ jobs:
           just coordinator initializer port-forwarder openssl service-mesh-proxy node-installer
       - name: E2E Test
         run: |
+          nix run .#scripts.get-logs workspace/e2e.namespace &
           nix shell -L .#contrast.e2e --command ${{ matrix.test_name }}.test -test.v \
             --image-replacements workspace/just.containerlookup \
             --namespace-file workspace/e2e.namespace \
             --platform aks-clh-snp \
             --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}"
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        with:
+          name: e2e_pod_logs-${{ matrix.test_name }}
+          path: workspace/namespace-logs
       - name: Cleanup
         if: cancelled() && !inputs.skip-undeploy
         run: |

diff --git a/.github/workflows/e2e_openssl_baremetal.yml b/.github/workflows/e2e_openssl_baremetal.yml
@@ -50,11 +50,18 @@ jobs:
           just coordinator initializer openssl port-forwarder node-installer K3s-QEMU-${{ matrix.tee }}
       - name: E2E Test
         run: |
+          nix run .#scripts.get-logs workspace/e2e.namespace &
           nix shell .#contrast.e2e --command openssl.test -test.v \
             --image-replacements workspace/just.containerlookup \
             --namespace-file workspace/e2e.namespace \
             --platform K3s-QEMU-${{ matrix.tee }} \
             --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}"
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        with:
+          name: e2e_pod_logs-${{ matrix.tee }}-${{ matrix.test_name }}
+          path: workspace/namespace-logs
       - name: Cleanup
         if: cancelled() && !inputs.skip-undeploy
         run: |

diff --git a/.github/workflows/e2e_regression.yml b/.github/workflows/e2e_regression.yml
@@ -70,11 +70,18 @@ jobs:
           fi
       - name: Run regression test
         run: |
+          nix run .#scripts.get-logs workspace/e2e.namespace &
           nix shell -L .#contrast.e2e --command ${{ matrix.case }}.test -test.v \
             --image-replacements workspace/just.containerlookup \
             --namespace-file workspace/e2e.namespace \
             --platform aks-clh-snp \
             --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}"
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        with:
+          name: e2e_pod_logs-${{ matrix.case }}
+          path: workspace/namespace-logs
       - name: Cleanup
         if: cancelled() && !inputs.skip-undeploy
         run: |

diff --git a/packages/scripts.nix b/packages/scripts.nix
@@ -317,4 +317,34 @@
       mv "$mergedConfig" "''${KUBECONFIG_BAK%%:*}"
     '';
   };
+
+  # Usage: get-logs $namespaceFile
+  get-logs = writeShellApplication {
+    name = "get-logs";
+    runtimeInputs = with pkgs; [ kubectl ];
+    text = ''
+      set -euo pipefail
+      # wait until namespace file is populated
+      while ! [[ -s "$1" ]]; do
+        sleep 1
+      done
+      namespace="$(head -n1 "$1")"
+      while kubectl get ns "$namespace" 1>/dev/null 2>/dev/null; do
+        pods="$(kubectl get pods -n "$namespace" | awk '!/^NAME/{print $1}')"
+        mkdir -p "workspace/namespace-logs"
+        for pod in $pods; do
+          logfile="workspace/namespace-logs/$pod.log"
+          if ! [[ -f "$logfile" ]]; then
+            {
+              touch "$logfile" # prevents creation of to much processes
+              # wait for all containers of the pod to come online, then collect the logs
+              kubectl wait pod --all --for=condition=Ready --timeout="-1s" -n "$namespace" "$pod" 1>/dev/null 2>/dev/null
+              kubectl logs -f --all-containers=true -n "$namespace" "$pod" > "$logfile"
+            } &
+          fi
+        done
+      done
+      wait
+    '';
+  };
 }