diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d92d10db541..597564c90ea 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -205,6 +205,7 @@ steps: KIND_VERSION: "v0.20.0" command: ".buildkite/scripts/steps/k8s-extended-tests.sh" artifact_paths: + - "build/k8s-logs*/*" - "build/TEST-**" - "build/diagnostics/*" agents: diff --git a/pkg/testing/runner/kubernetes.go b/pkg/testing/runner/kubernetes.go index 4c998a5e4df..5a8fd2c3294 100644 --- a/pkg/testing/runner/kubernetes.go +++ b/pkg/testing/runner/kubernetes.go @@ -6,6 +6,7 @@ package runner import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -63,6 +64,19 @@ func (KubernetesRunner) Run(ctx context.Context, verbose bool, sshClient SSHClie env["AGENT_VERSION"] = agentVersion env["TEST_DEFINE_PREFIX"] = testPrefix + buildFolderAbsPath, err := filepath.Abs("build") + if err != nil { + return OSRunnerResult{}, err + } + + podLogsPath := filepath.Join(buildFolderAbsPath, fmt.Sprintf("k8s-logs-%s", testPrefix)) + err = os.Mkdir(podLogsPath, 0755) + if err != nil && !errors.Is(err, os.ErrExist) { + return OSRunnerResult{}, err + } + + env["K8S_TESTS_POD_LOGS_BASE"] = podLogsPath + params := devtools.GoTestArgs{ LogName: testName, OutputFile: fileName + ".out", @@ -72,7 +86,7 @@ func (KubernetesRunner) Run(ctx context.Context, verbose bool, sshClient SSHClie ExtraFlags: extraFlags, Env: env, } - err := devtools.GoTest(ctx, params) + err = devtools.GoTest(ctx, params) if err != nil { return OSRunnerResult{}, err } diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/kubernetes_agent_standalone_test.go index 6ac1b73632f..41f06b28cc8 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/kubernetes_agent_standalone_test.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "os" + "path/filepath" "testing" "time" @@ -26,6 +27,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/e2e-framework/klient" "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/kustomize/api/krusty" "sigs.k8s.io/kustomize/kyaml/filesys" @@ -51,6 +54,9 @@ func TestKubernetesAgentStandalone(t *testing.T) { require.NoError(t, err) require.NotNil(t, client) + testLogsBasePath := os.Getenv("K8S_TESTS_POD_LOGS_BASE") + require.NotEmpty(t, testLogsBasePath) + ctx := context.Background() namespace := info.Namespace @@ -79,6 +85,9 @@ func TestKubernetesAgentStandalone(t *testing.T) { }, } t.Cleanup(func() { + if t.Failed() { + dumpLogs(t, ctx, client, namespace, testLogsBasePath) + } _ = client.Resources().Delete(ctx, k8sNamespaceObj) for _, obj := range objects { _ = client.Resources(namespace).Delete(ctx, obj) @@ -129,6 +138,12 @@ func TestKubernetesAgentStandalone(t *testing.T) { require.NoError(t, err) for _, pod := range podList.Items { + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.RestartCount > 0 { + return false + } + } + for _, cond := range pod.Status.Conditions { if cond.Type != corev1.PodReady { continue @@ -141,8 +156,63 @@ func TestKubernetesAgentStandalone(t *testing.T) { } return true - }, time.Second*100, time.Second*1) - require.NoError(t, err) + }, time.Second*100, time.Second*1, "Timed out waiting for pods to be ready") +} + +func dumpLogs(t *testing.T, ctx context.Context, client klient.Client, namespace string, targetDir string) { + + podList := &corev1.PodList{} + + clientset, err := kubernetes.NewForConfig(client.RESTConfig()) + if err != nil { + t.Logf("Error creating clientset: %v\n", err) + return + } + + err = client.Resources(namespace).List(ctx, podList) + if err != nil { + t.Logf("Error listing pods: %v\n", err) + return + } + + for _, pod := range podList.Items { + + previous := false + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.RestartCount > 0 { + previous = true + break + } + } + + for _, container := range pod.Spec.Containers { + logFilePath := filepath.Join(targetDir, fmt.Sprintf("%s-%s-%s.log", t.Name(), pod.Name, container.Name)) + logFile, err := os.Create(logFilePath) + if err != nil { + t.Logf("Error creating log file: %v\n", err) + continue + } + + req := clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: container.Name, + Previous: previous, + }) + podLogsStream, err := req.Stream(context.TODO()) + if err != nil { + t.Logf("Error getting container %s of pod %s logs: %v\n", container.Name, pod.Name, err) + continue + } + + _, err = io.Copy(logFile, podLogsStream) + if err != nil { + t.Logf("Error writing container %s of pod %s logs: %v\n", container.Name, pod.Name, err) + } else { + t.Logf("Wrote container %s of pod %s logs to %s\n", container.Name, pod.Name, logFilePath) + } + + _ = podLogsStream.Close() + } + } } // YAMLDecoder converts YAML bytes into test.Builder instances.