Improve debug output for K8s tests

.buildkite/pipeline.yml

@@ -205,6 +205,7 @@ steps:
           KIND_VERSION: "v0.20.0"
         command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
         artifact_paths:
+          - "build/k8s-logs*/*"
           - "build/TEST-**"
           - "build/diagnostics/*"
         agents:

pkg/testing/runner/kubernetes.go

@@ -6,6 +6,7 @@ package runner
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -63,6 +64,19 @@ func (KubernetesRunner) Run(ctx context.Context, verbose bool, sshClient SSHClie
 		env["AGENT_VERSION"] = agentVersion
 		env["TEST_DEFINE_PREFIX"] = testPrefix
 
+		buildFolderAbsPath, err := filepath.Abs("build")
+		if err != nil {
+			return OSRunnerResult{}, err
+		}
+
+		podLogsPath := filepath.Join(buildFolderAbsPath, fmt.Sprintf("k8s-logs-%s", testPrefix))
+		err = os.Mkdir(podLogsPath, 0755)
+		if err != nil && !errors.Is(err, os.ErrExist) {
+			return OSRunnerResult{}, err
+		}
+
+		env["K8S_TESTS_POD_LOGS_BASE"] = podLogsPath
+
 		params := devtools.GoTestArgs{
 			LogName:         testName,
 			OutputFile:      fileName + ".out",
@@ -72,7 +86,7 @@ func (KubernetesRunner) Run(ctx context.Context, verbose bool, sshClient SSHClie
 			ExtraFlags:      extraFlags,
 			Env:             env,
 		}
-		err := devtools.GoTest(ctx, params)
+		err = devtools.GoTest(ctx, params)
 		if err != nil {
 			return OSRunnerResult{}, err
 		}

testing/integration/kubernetes_agent_standalone_test.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -26,6 +27,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/e2e-framework/klient"
 	"sigs.k8s.io/e2e-framework/klient/k8s"
 	"sigs.k8s.io/kustomize/api/krusty"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
@@ -51,6 +54,9 @@ func TestKubernetesAgentStandalone(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, client)
 
+	testLogsBasePath := os.Getenv("K8S_TESTS_POD_LOGS_BASE")
+	require.NotEmpty(t, testLogsBasePath)
+
 	ctx := context.Background()
 
 	namespace := info.Namespace
@@ -79,6 +85,9 @@ func TestKubernetesAgentStandalone(t *testing.T) {
 		},
 	}
 	t.Cleanup(func() {
+		if t.Failed() {
+			dumpLogs(t, ctx, client, namespace, testLogsBasePath)
+		}
 		_ = client.Resources().Delete(ctx, k8sNamespaceObj)
 		for _, obj := range objects {
 			_ = client.Resources(namespace).Delete(ctx, obj)
@@ -129,6 +138,12 @@ func TestKubernetesAgentStandalone(t *testing.T) {
 		require.NoError(t, err)
 
 		for _, pod := range podList.Items {
+			for _, containerStatus := range pod.Status.ContainerStatuses {
+				if containerStatus.RestartCount > 0 {
+					return false
+				}
+			}
+
 			for _, cond := range pod.Status.Conditions {
 				if cond.Type != corev1.PodReady {
 					continue
@@ -141,8 +156,63 @@ func TestKubernetesAgentStandalone(t *testing.T) {
 		}
 
 		return true
-	}, time.Second*100, time.Second*1)
-	require.NoError(t, err)
+	}, time.Second*100, time.Second*1, "Timed out waiting for pods to be ready")
+}
+
+func dumpLogs(t *testing.T, ctx context.Context, client klient.Client, namespace string, targetDir string) {
+
+	podList := &corev1.PodList{}
+
+	clientset, err := kubernetes.NewForConfig(client.RESTConfig())
+	if err != nil {
+		t.Logf("Error creating clientset: %v\n", err)
+		return
+	}
+
+	err = client.Resources(namespace).List(ctx, podList)
+	if err != nil {
+		t.Logf("Error listing pods: %v\n", err)
+		return
+	}
+
+	for _, pod := range podList.Items {
+
+		previous := false
+		for _, containerStatus := range pod.Status.ContainerStatuses {
+			if containerStatus.RestartCount > 0 {
+				previous = true
+				break
+			}
+		}
+
+		for _, container := range pod.Spec.Containers {
+			logFilePath := filepath.Join(targetDir, fmt.Sprintf("%s-%s-%s.log", t.Name(), pod.Name, container.Name))
+			logFile, err := os.Create(logFilePath)
+			if err != nil {
+				t.Logf("Error creating log file: %v\n", err)
+				continue
+			}
+
+			req := clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
+				Container: container.Name,
+				Previous:  previous,
+			})
+			podLogsStream, err := req.Stream(context.TODO())
+			if err != nil {
+				t.Logf("Error getting container %s of pod %s logs: %v\n", container.Name, pod.Name, err)
+				continue
+			}
+
+			_, err = io.Copy(logFile, podLogsStream)
+			if err != nil {
+				t.Logf("Error writing container %s of pod %s logs: %v\n", container.Name, pod.Name, err)
+			} else {
+				t.Logf("Wrote container %s of pod %s logs to %s\n", container.Name, pod.Name, logFilePath)
+			}
+
+			_ = podLogsStream.Close()
+		}
+	}
 }
 
 // YAMLDecoder converts YAML bytes into test.Builder instances.