From a15abec8b75eb5c46f584491b757f8ccc59aff93 Mon Sep 17 00:00:00 2001 From: miampf Date: Fri, 16 Aug 2024 15:17:40 +0200 Subject: [PATCH 1/6] shell script for collecting logs improved script log path easily predictable --- packages/scripts.nix | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/scripts.nix b/packages/scripts.nix index 5b3478dffa..fbccb0fb47 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -317,4 +317,34 @@ mv "$mergedConfig" "''${KUBECONFIG_BAK%%:*}" ''; }; + + # Usage: get-logs $namespaceFile + get-logs = writeShellApplication { + name = "get-logs"; + runtimeInputs = with pkgs; [ kubectl ]; + text = '' + set -euo pipefail + # wait until namespace file is populated + while ! [[ -s "$1" ]]; do + sleep 1 + done + namespace="$(head -n1 "$1")" + while kubectl get ns "$namespace"; do + pods="$(kubectl get pods -n "$namespace" | awk '!/^NAME/{print $1}')" + mkdir -p "workspace/namespace-logs" + while IFS= read -r pod; do + logfile="workspace/namespace-logs/$pod.log" + if ! [[ -f "$logfile" ]]; then + { + touch "$logfile" # prevents creation of to much processes + # wait for all containers of the pod to come online, then collect the logs + kubectl wait pod --all --for=condition=Ready --timeout="-1s" -n "$namespace" "$pod" + kubectl logs -f --all-containers=true -n "$namespace" "$pod" > "$logfile" + } & + fi + done <<< "$pods" + done + wait + ''; + }; } From f3b962d5ad926e0911dc4fed6d434e430afc4e5e Mon Sep 17 00:00:00 2001 From: miampf Date: Thu, 22 Aug 2024 15:21:55 +0200 Subject: [PATCH 2/6] add logic for openssl e2e test adjust workflow correct namespace file always upload logs --- .github/workflows/e2e_openssl_baremetal.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/e2e_openssl_baremetal.yml b/.github/workflows/e2e_openssl_baremetal.yml index 412b8a44cd..0abd593d0e 100644 --- a/.github/workflows/e2e_openssl_baremetal.yml +++ b/.github/workflows/e2e_openssl_baremetal.yml @@ -50,11 +50,18 @@ jobs: just coordinator initializer openssl port-forwarder node-installer K3s-QEMU-${{ matrix.tee }} - name: E2E Test run: | + nix run .#scripts.get-logs workspace/e2e.namespace 1> /dev/null 2> /dev/null & nix shell .#contrast.e2e --command openssl.test -test.v \ --image-replacements workspace/just.containerlookup \ --namespace-file workspace/e2e.namespace \ --platform K3s-QEMU-${{ matrix.tee }} \ --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}" + - name: Upload logs + if: always() + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a + with: + name: e2e_pod_logs + path: workspace/namespace-logs - name: Cleanup if: cancelled() && !inputs.skip-undeploy run: | From 152c33bf0cc82a889d077eccab179d3d5274ae5e Mon Sep 17 00:00:00 2001 From: miampf Date: Fri, 23 Aug 2024 12:56:09 +0200 Subject: [PATCH 3/6] make script output nothing to console by itself --- .github/workflows/e2e_openssl_baremetal.yml | 2 +- packages/scripts.nix | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_openssl_baremetal.yml b/.github/workflows/e2e_openssl_baremetal.yml index 0abd593d0e..e8dc005221 100644 --- a/.github/workflows/e2e_openssl_baremetal.yml +++ b/.github/workflows/e2e_openssl_baremetal.yml @@ -50,7 +50,7 @@ jobs: just coordinator initializer openssl port-forwarder node-installer K3s-QEMU-${{ matrix.tee }} - name: E2E Test run: | - nix run .#scripts.get-logs workspace/e2e.namespace 1> /dev/null 2> /dev/null & + nix run .#scripts.get-logs workspace/e2e.namespace & nix shell .#contrast.e2e --command openssl.test -test.v \ --image-replacements workspace/just.containerlookup \ --namespace-file workspace/e2e.namespace \ diff --git a/packages/scripts.nix b/packages/scripts.nix index fbccb0fb47..8f08c0d0f4 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -329,7 +329,7 @@ sleep 1 done namespace="$(head -n1 "$1")" - while kubectl get ns "$namespace"; do + while kubectl get ns "$namespace" 1>/dev/null 2>/dev/null; do pods="$(kubectl get pods -n "$namespace" | awk '!/^NAME/{print $1}')" mkdir -p "workspace/namespace-logs" while IFS= read -r pod; do @@ -338,7 +338,7 @@ { touch "$logfile" # prevents creation of to much processes # wait for all containers of the pod to come online, then collect the logs - kubectl wait pod --all --for=condition=Ready --timeout="-1s" -n "$namespace" "$pod" + kubectl wait pod --all --for=condition=Ready --timeout="-1s" -n "$namespace" "$pod" 1>/dev/null 2>/dev/null kubectl logs -f --all-containers=true -n "$namespace" "$pod" > "$logfile" } & fi From 906729949ba65ed83638a8492b9edb89b83469e7 Mon Sep 17 00:00:00 2001 From: miampf Date: Fri, 23 Aug 2024 13:00:16 +0200 Subject: [PATCH 4/6] implement for other e2e tests correct name --- .github/workflows/e2e_kubernetes.yaml | 7 +++++++ .github/workflows/e2e_regression.yml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/e2e_kubernetes.yaml b/.github/workflows/e2e_kubernetes.yaml index f49ada8b2a..952e055431 100644 --- a/.github/workflows/e2e_kubernetes.yaml +++ b/.github/workflows/e2e_kubernetes.yaml @@ -69,11 +69,18 @@ jobs: just coordinator initializer port-forwarder openssl service-mesh-proxy node-installer - name: E2E Test run: | + nix run .#scripts.get-logs workspace/e2e.namespace & nix shell -L .#contrast.e2e --command ${{ matrix.test_name }}.test -test.v \ --image-replacements workspace/just.containerlookup \ --namespace-file workspace/e2e.namespace \ --platform aks-clh-snp \ --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}" + - name: Upload logs + if: always() + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a + with: + name: e2e_pod_logs-${{ matrix.test_name }} + path: workspace/namespace-logs - name: Cleanup if: cancelled() && !inputs.skip-undeploy run: | diff --git a/.github/workflows/e2e_regression.yml b/.github/workflows/e2e_regression.yml index 5a5eb3e40b..2bee2095a5 100644 --- a/.github/workflows/e2e_regression.yml +++ b/.github/workflows/e2e_regression.yml @@ -70,11 +70,18 @@ jobs: fi - name: Run regression test run: | + nix run .#scripts.get-logs workspace/e2e.namespace & nix shell -L .#contrast.e2e --command ${{ matrix.case }}.test -test.v \ --image-replacements workspace/just.containerlookup \ --namespace-file workspace/e2e.namespace \ --platform aks-clh-snp \ --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}" + - name: Upload logs + if: always() + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a + with: + name: e2e_pod_logs-${{ matrix.case }} + path: workspace/namespace-logs - name: Cleanup if: cancelled() && !inputs.skip-undeploy run: | From a6d3bada67996deb8ef0ae692b92fcf1827c6bcb Mon Sep 17 00:00:00 2001 From: miampf Date: Wed, 28 Aug 2024 13:06:19 +0200 Subject: [PATCH 5/6] use simpler for loop --- packages/scripts.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/scripts.nix b/packages/scripts.nix index 8f08c0d0f4..d0082fc8a9 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -332,7 +332,7 @@ while kubectl get ns "$namespace" 1>/dev/null 2>/dev/null; do pods="$(kubectl get pods -n "$namespace" | awk '!/^NAME/{print $1}')" mkdir -p "workspace/namespace-logs" - while IFS= read -r pod; do + for pod in $pods; do logfile="workspace/namespace-logs/$pod.log" if ! [[ -f "$logfile" ]]; then { @@ -342,7 +342,7 @@ kubectl logs -f --all-containers=true -n "$namespace" "$pod" > "$logfile" } & fi - done <<< "$pods" + done done wait ''; From 0b42eb0c5082d7d033b972d6229e4e50097cdc35 Mon Sep 17 00:00:00 2001 From: miampf Date: Thu, 29 Aug 2024 13:30:21 +0200 Subject: [PATCH 6/6] fix up artifact name after rebase --- .github/workflows/e2e_openssl_baremetal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_openssl_baremetal.yml b/.github/workflows/e2e_openssl_baremetal.yml index e8dc005221..117e4b295b 100644 --- a/.github/workflows/e2e_openssl_baremetal.yml +++ b/.github/workflows/e2e_openssl_baremetal.yml @@ -60,7 +60,7 @@ jobs: if: always() uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a with: - name: e2e_pod_logs + name: e2e_pod_logs-${{ matrix.tee }}-${{ matrix.test_name }} path: workspace/namespace-logs - name: Cleanup if: cancelled() && !inputs.skip-undeploy