From 86402286caf565bef8db70978b1214a60d481827 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 00:21:29 +0200 Subject: [PATCH] [k8s]: transition integration tests to adapter pattern (#6277) (#6359) * feat: transition kubernetes integration tests to adapter pattern * fix: count nodes without NoSchedule taints (cherry picked from commit 67c744fd51e95401ce8de96d27278bb7ed3d3cde) Co-authored-by: Panos Koutsovasilis --- .../kubernetes_agent_service_test.go | 91 +- .../kubernetes_agent_standalone_test.go | 872 +++++++++--------- 2 files changed, 485 insertions(+), 478 deletions(-) diff --git a/testing/integration/kubernetes_agent_service_test.go b/testing/integration/kubernetes_agent_service_test.go index ed479c99702..a66d8547d8a 100644 --- a/testing/integration/kubernetes_agent_service_test.go +++ b/testing/integration/kubernetes_agent_service_test.go @@ -7,17 +7,13 @@ package integration import ( - "bufio" - "bytes" "context" - "fmt" "os" "path/filepath" "testing" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/e2e-framework/klient/k8s" "github.com/elastic/elastic-agent/pkg/testing/define" @@ -35,73 +31,38 @@ func TestKubernetesAgentService(t *testing.T) { Group: define.Kubernetes, }) + // read the service agent config + serviceAgentYAML, err := os.ReadFile(filepath.Join("testdata", "connectors.agent.yml")) + require.NoError(t, err, "failed to read service agent config") + ctx := context.Background() kCtx := k8sGetContext(t, info) - testNamespace := kCtx.getNamespace(t) - - renderedManifest, err := renderKustomize(agentK8SKustomize) - require.NoError(t, err, "failed to render kustomize") - - k8sObjects, err := k8sYAMLToObjects(bufio.NewReader(bytes.NewReader(renderedManifest))) - require.NoError(t, err, "failed to convert yaml to k8s objects") - // add the testNamespace in the k8sObjects - k8sObjects = append([]k8s.Object{&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}}, k8sObjects...) + schedulableNodeCount, err := k8sSchedulableNodeCount(ctx, kCtx) + require.NoError(t, err, "error at getting schedulable node count") + require.NotZero(t, schedulableNodeCount, "no schedulable Kubernetes nodes found") - t.Cleanup(func() { - err = k8sDeleteObjects(ctx, kCtx.client, k8sDeleteOpts{wait: true}, k8sObjects...) - require.NoError(t, err, "failed to delete k8s namespace") - }) - - k8sKustomizeAdjustObjects(k8sObjects, testNamespace, "elastic-agent-standalone", - func(container *corev1.Container) { - // set agent image - container.Image = kCtx.agentImage - // set ImagePullPolicy to "Never" to avoid pulling the image - // as the image is already loaded by the kubernetes provisioner - container.ImagePullPolicy = "Never" - - // set Elasticsearch host and API key - for idx, env := range container.Env { - if env.Name == "ES_HOST" { - container.Env[idx].Value = kCtx.esHost - container.Env[idx].ValueFrom = nil - } - if env.Name == "API_KEY" { - container.Env[idx].Value = kCtx.esAPIKey - container.Env[idx].ValueFrom = nil + testSteps := []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepDeployKustomize(agentK8SKustomize, "elastic-agent-standalone", k8sKustomizeOverrides{ + agentContainerMemoryLimit: "800Mi", + }, func(obj k8s.Object) { + // update the configmap to only run the connectors input + switch objWithType := obj.(type) { + case *corev1.ConfigMap: + _, ok := objWithType.Data["agent.yml"] + if ok { + objWithType.Data["agent.yml"] = string(serviceAgentYAML) } } - }, - func(pod *corev1.PodSpec) { - for volumeIdx, volume := range pod.Volumes { - // need to update the volume path of the state directory - // to match the test namespace - if volume.Name == "elastic-agent-state" { - hostPathType := corev1.HostPathDirectoryOrCreate - pod.Volumes[volumeIdx].VolumeSource.HostPath = &corev1.HostPathVolumeSource{ - Type: &hostPathType, - Path: fmt.Sprintf("/var/lib/elastic-agent-standalone/%s/state", testNamespace), - } - } - } - }) - - // update the configmap to only run the connectors input - serviceAgentYAML, err := os.ReadFile(filepath.Join("testdata", "connectors.agent.yml")) - require.NoError(t, err) - for _, obj := range k8sObjects { - switch objWithType := obj.(type) { - case *corev1.ConfigMap: - _, ok := objWithType.Data["agent.yml"] - if ok { - objWithType.Data["agent.yml"] = string(serviceAgentYAML) - } - } + }), + k8sStepCheckAgentStatus("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone", map[string]bool{ + "connectors-py": true, + }), } - k8sKustomizeDeployAgent(t, ctx, kCtx.client, k8sObjects, testNamespace, false, kCtx.logsBasePath, - true, map[string]bool{ - "connectors-py": true, - }) + testNamespace := kCtx.getNamespace(t) + for _, step := range testSteps { + step(t, ctx, kCtx, testNamespace) + } } diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/kubernetes_agent_standalone_test.go index 86db561edd8..148706f3654 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/kubernetes_agent_standalone_test.go @@ -77,64 +77,84 @@ func TestKubernetesAgentStandaloneKustomize(t *testing.T) { Group: define.Kubernetes, }) + ctx := context.Background() kCtx := k8sGetContext(t, info) - renderedManifest, err := renderKustomize(agentK8SKustomize) - require.NoError(t, err, "failed to render kustomize") + schedulableNodeCount, err := k8sSchedulableNodeCount(ctx, kCtx) + require.NoError(t, err, "error at getting schedulable node count") + require.NotZero(t, schedulableNodeCount, "no schedulable Kubernetes nodes found") testCases := []struct { - name string - runUser *int64 - runGroup *int64 - capabilitiesDrop []corev1.Capability - capabilitiesAdd []corev1.Capability - runK8SInnerTests bool - skipReason string + name string + skipReason string + steps []k8sTestStep }{ { - "default deployment - rootful agent", - nil, - nil, - nil, - nil, - false, - "", + name: "default deployment - rootful agent", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepDeployKustomize(agentK8SKustomize, "elastic-agent-standalone", k8sKustomizeOverrides{ + agentContainerMemoryLimit: "800Mi", + }, nil), + k8sStepCheckAgentStatus("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone", nil), + }, }, { - "drop ALL capabilities - rootful agent", - int64Ptr(0), - nil, - []corev1.Capability{"ALL"}, - []corev1.Capability{}, - false, - "", + name: "drop ALL capabilities - rootful agent", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepDeployKustomize(agentK8SKustomize, "elastic-agent-standalone", k8sKustomizeOverrides{ + agentContainerRunUser: int64Ptr(0), + agentContainerCapabilitiesAdd: []corev1.Capability{}, + agentContainerCapabilitiesDrop: []corev1.Capability{"ALL"}, + agentContainerMemoryLimit: "800Mi", + }, nil), + k8sStepCheckAgentStatus("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone", nil), + }, }, { - "drop ALL add CHOWN, SETPCAP capabilities - rootful agent", - int64Ptr(0), - nil, - []corev1.Capability{"ALL"}, - []corev1.Capability{"CHOWN", "SETPCAP"}, - true, - "", + name: "drop ALL add CHOWN, SETPCAP capabilities - rootful agent", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepDeployKustomize(agentK8SKustomize, "elastic-agent-standalone", k8sKustomizeOverrides{ + agentContainerRunUser: int64Ptr(0), + agentContainerCapabilitiesAdd: []corev1.Capability{"CHOWN", "SETPCAP"}, + agentContainerCapabilitiesDrop: []corev1.Capability{"ALL"}, + agentContainerMemoryLimit: "800Mi", + }, nil), + k8sStepCheckAgentStatus("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone", nil), + k8sStepRunInnerTests("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone"), + }, }, { - "drop ALL add CHOWN, SETPCAP capabilities - rootless agent", - int64Ptr(1000), // elastic-agent uid - nil, - []corev1.Capability{"ALL"}, - []corev1.Capability{"CHOWN", "SETPCAP", "DAC_READ_SEARCH", "SYS_PTRACE"}, - true, - "", + name: "drop ALL add CHOWN, SETPCAP, DAC_READ_SEARCH, SYS_PTRACE capabilities - rootless agent", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepDeployKustomize(agentK8SKustomize, "elastic-agent-standalone", k8sKustomizeOverrides{ + agentContainerRunUser: int64Ptr(1000), + agentContainerRunGroup: int64Ptr(1000), + agentContainerCapabilitiesAdd: []corev1.Capability{"CHOWN", "SETPCAP", "DAC_READ_SEARCH", "SYS_PTRACE"}, + agentContainerCapabilitiesDrop: []corev1.Capability{"ALL"}, + agentContainerMemoryLimit: "800Mi", + }, nil), + k8sStepCheckAgentStatus("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone", nil), + k8sStepRunInnerTests("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone"), + }, }, { - "drop ALL add CHOWN, SETPCAP capabilities - rootless agent random uid:gid", - int64Ptr(500), - int64Ptr(500), - []corev1.Capability{"ALL"}, - []corev1.Capability{"CHOWN", "SETPCAP", "DAC_READ_SEARCH", "SYS_PTRACE"}, - true, - "", + name: "drop ALL add CHOWN, SETPCAP, DAC_READ_SEARCH, SYS_PTRACE capabilities - rootless agent random uid:gid", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepDeployKustomize(agentK8SKustomize, "elastic-agent-standalone", k8sKustomizeOverrides{ + agentContainerRunUser: int64Ptr(500), + agentContainerRunGroup: int64Ptr(500), + agentContainerCapabilitiesAdd: []corev1.Capability{"CHOWN", "SETPCAP", "DAC_READ_SEARCH", "SYS_PTRACE"}, + agentContainerCapabilitiesDrop: []corev1.Capability{"ALL"}, + agentContainerMemoryLimit: "800Mi", + }, nil), + k8sStepCheckAgentStatus("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone", nil), + k8sStepRunInnerTests("app=elastic-agent-standalone", schedulableNodeCount, "elastic-agent-standalone"), + }, }, } @@ -145,72 +165,11 @@ func TestKubernetesAgentStandaloneKustomize(t *testing.T) { t.Skip(tc.skipReason) } - ctx := context.Background() - testNamespace := kCtx.getNamespace(t) - k8sObjects, err := k8sYAMLToObjects(bufio.NewReader(bytes.NewReader(renderedManifest))) - require.NoError(t, err, "failed to convert yaml to k8s objects") - - // add the testNamespace in the beginning of k8sObjects to be created first - k8sObjects = append([]k8s.Object{&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}}, k8sObjects...) - - t.Cleanup(func() { - err = k8sDeleteObjects(ctx, kCtx.client, k8sDeleteOpts{wait: true}, k8sObjects...) - require.NoError(t, err, "failed to delete k8s namespace") - }) - - k8sKustomizeAdjustObjects(k8sObjects, testNamespace, "elastic-agent-standalone", - func(container *corev1.Container) { - // set agent image - container.Image = kCtx.agentImage - // set ImagePullPolicy to "Never" to avoid pulling the image - // as the image is already loaded by the kubernetes provisioner - container.ImagePullPolicy = "Never" - - container.Resources.Limits = corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("800Mi"), - } - - if tc.capabilitiesDrop != nil || tc.capabilitiesAdd != nil || tc.runUser != nil || tc.runGroup != nil { - // set security context - container.SecurityContext = &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Drop: tc.capabilitiesDrop, - Add: tc.capabilitiesAdd, - }, - RunAsUser: tc.runUser, - RunAsGroup: tc.runGroup, - } - } - // set Elasticsearch host and API key - for idx, env := range container.Env { - if env.Name == "ES_HOST" { - container.Env[idx].Value = kCtx.esHost - container.Env[idx].ValueFrom = nil - } - if env.Name == "API_KEY" { - container.Env[idx].Value = kCtx.esAPIKey - container.Env[idx].ValueFrom = nil - } - } - }, - func(pod *corev1.PodSpec) { - for volumeIdx, volume := range pod.Volumes { - // need to update the volume path of the state directory - // to match the test namespace - if volume.Name == "elastic-agent-state" { - hostPathType := corev1.HostPathDirectoryOrCreate - pod.Volumes[volumeIdx].VolumeSource.HostPath = &corev1.HostPathVolumeSource{ - Type: &hostPathType, - Path: fmt.Sprintf("/var/lib/elastic-agent-standalone/%s/state", testNamespace), - } - } - } - }) - - k8sKustomizeDeployAgent(t, ctx, kCtx.client, k8sObjects, testNamespace, tc.runK8SInnerTests, - kCtx.logsBasePath, true, nil) + for _, step := range tc.steps { + step(t, ctx, kCtx, testNamespace) + } }) } } @@ -228,85 +187,47 @@ func TestKubernetesAgentOtel(t *testing.T) { Group: define.Kubernetes, }) + ctx := context.Background() kCtx := k8sGetContext(t, info) - renderedManifest, err := renderKustomize(agentK8SKustomize) - require.NoError(t, err, "failed to render kustomize") + nodeList := corev1.NodeList{} + err := kCtx.client.Resources().List(ctx, &nodeList) + require.NoError(t, err) + + schedulableNodeCount, err := k8sSchedulableNodeCount(ctx, kCtx) + require.NoError(t, err, "error at getting schedulable node count") + require.NotZero(t, schedulableNodeCount, "no schedulable Kubernetes nodes found") testCases := []struct { - name string - envAdd []corev1.EnvVar - runK8SInnerTests bool + name string + skipReason string + steps []k8sTestStep }{ { - "run agent in otel mode", - []corev1.EnvVar{ - {Name: "ELASTIC_AGENT_OTEL", Value: "true"}, + name: "run agent in otel mode", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepDeployKustomize(agentK8SKustomize, "elastic-agent-standalone", k8sKustomizeOverrides{ + agentContainerMemoryLimit: "800Mi", + agentContainerExtraEnv: []corev1.EnvVar{{Name: "ELASTIC_AGENT_OTEL", Value: "true"}}, + agentContainerArgs: []string{}, // clear default args + }, nil), }, - false, }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - ctx := context.Background() - testNamespace := kCtx.getNamespace(t) - - k8sObjects, err := k8sYAMLToObjects(bufio.NewReader(bytes.NewReader(renderedManifest))) - require.NoError(t, err, "failed to convert yaml to k8s objects") - - // add the testNamespace in the k8sObjects - k8sObjects = append([]k8s.Object{&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}}, k8sObjects...) - - t.Cleanup(func() { - err = k8sDeleteObjects(ctx, kCtx.client, k8sDeleteOpts{wait: true}, k8sObjects...) - require.NoError(t, err, "failed to delete k8s namespace") - }) - - k8sKustomizeAdjustObjects(k8sObjects, testNamespace, "elastic-agent-standalone", - func(container *corev1.Container) { - // set agent image - container.Image = kCtx.agentImage - // set ImagePullPolicy to "Never" to avoid pulling the image - // as the image is already loaded by the kubernetes provisioner - container.ImagePullPolicy = "Never" - - // set Elasticsearch host and API key - for idx, env := range container.Env { - if env.Name == "ES_HOST" { - container.Env[idx].Value = kCtx.esHost - container.Env[idx].ValueFrom = nil - } - if env.Name == "API_KEY" { - container.Env[idx].Value = kCtx.esAPIKey - container.Env[idx].ValueFrom = nil - } - } - - if len(tc.envAdd) > 0 { - container.Env = append(container.Env, tc.envAdd...) - } + if tc.skipReason != "" { + t.Skip(tc.skipReason) + } - // drop arguments overriding default config - container.Args = []string{} - }, - func(pod *corev1.PodSpec) { - for volumeIdx, volume := range pod.Volumes { - // need to update the volume path of the state directory - // to match the test namespace - if volume.Name == "elastic-agent-state" { - hostPathType := corev1.HostPathDirectoryOrCreate - pod.Volumes[volumeIdx].VolumeSource.HostPath = &corev1.HostPathVolumeSource{ - Type: &hostPathType, - Path: fmt.Sprintf("/var/lib/elastic-agent-standalone/%s/state", testNamespace), - } - } - } - }) + testNamespace := kCtx.getNamespace(t) - k8sKustomizeDeployAgent(t, ctx, kCtx.client, k8sObjects, testNamespace, - false, kCtx.logsBasePath, false, nil) + for _, step := range tc.steps { + step(t, ctx, kCtx, testNamespace) + } }) } } @@ -331,282 +252,145 @@ func TestKubernetesAgentHelm(t *testing.T) { err := kCtx.client.Resources().List(ctx, &nodeList) require.NoError(t, err) - totalK8SNodes := len(nodeList.Items) - require.NotZero(t, totalK8SNodes, "No Kubernetes nodes found") + schedulableNodeCount, err := k8sSchedulableNodeCount(ctx, kCtx) + require.NoError(t, err, "error at getting schedulable node count") + require.NotZero(t, schedulableNodeCount, "no schedulable Kubernetes nodes found") testCases := []struct { - name string - values map[string]any - atLeastAgentPods int - runK8SInnerTests bool - agentPodLabelSelectors []string + name string + skipReason string + steps []k8sTestStep }{ { name: "helm standalone agent default kubernetes privileged", - values: map[string]any{ - "kubernetes": map[string]any{ - "enabled": true, - }, - "agent": map[string]any{ - "unprivileged": false, - "image": map[string]any{ - "repository": kCtx.agentImageRepo, - "tag": kCtx.agentImageTag, - "pullPolicy": "Never", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "kubernetes": map[string]any{ + "enabled": true, }, - }, - "outputs": map[string]any{ - "default": map[string]any{ - "type": "ESPlainAuthAPI", - "url": kCtx.esHost, - "api_key": kCtx.esAPIKey, + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, }, - }, - }, - runK8SInnerTests: true, - // - perNode Daemonset (totalK8SNodes pods) - // - clusterWide Deployment (1 agent pod) - // - ksmSharded Statefulset (1 agent pod) - atLeastAgentPods: totalK8SNodes + 1 + 1, - agentPodLabelSelectors: []string{ - // name=agent-{preset}-{release} - "name=agent-pernode-helm-agent", - "name=agent-clusterwide-helm-agent", - "name=agent-ksmsharded-helm-agent", + "outputs": map[string]any{ + "default": map[string]any{ + "type": "ESPlainAuthAPI", + "url": kCtx.esHost, + "api_key": kCtx.esAPIKey, + }, + }, + }), + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil), + k8sStepCheckAgentStatus("name=agent-clusterwide-helm-agent", 1, "agent", nil), + k8sStepCheckAgentStatus("name=agent-ksmsharded-helm-agent", 1, "agent", nil), + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent"), + k8sStepRunInnerTests("name=agent-clusterwide-helm-agent", 1, "agent"), + k8sStepRunInnerTests("name=agent-ksmsharded-helm-agent", 1, "agent"), }, }, { name: "helm standalone agent default kubernetes unprivileged", - values: map[string]any{ - "kubernetes": map[string]any{ - "enabled": true, - }, - "agent": map[string]any{ - "unprivileged": true, - "image": map[string]any{ - "repository": kCtx.agentImageRepo, - "tag": kCtx.agentImageTag, - "pullPolicy": "Never", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "kubernetes": map[string]any{ + "enabled": true, }, - }, - "outputs": map[string]any{ - "default": map[string]any{ - "type": "ESPlainAuthAPI", - "url": kCtx.esHost, - "api_key": kCtx.esAPIKey, + "agent": map[string]any{ + "unprivileged": true, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, }, - }, - }, - runK8SInnerTests: true, - // - perNode Daemonset (totalK8SNodes pods) - // - clusterWide Deployment (1 agent pod) - // - ksmSharded Statefulset (1 agent pod) - atLeastAgentPods: totalK8SNodes + 1 + 1, - agentPodLabelSelectors: []string{ - // name=agent-{preset}-{release} - "name=agent-pernode-helm-agent", - "name=agent-clusterwide-helm-agent", - "name=agent-ksmsharded-helm-agent", + "outputs": map[string]any{ + "default": map[string]any{ + "type": "ESPlainAuthAPI", + "url": kCtx.esHost, + "api_key": kCtx.esAPIKey, + }, + }, + }), + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil), + k8sStepCheckAgentStatus("name=agent-clusterwide-helm-agent", 1, "agent", nil), + k8sStepCheckAgentStatus("name=agent-ksmsharded-helm-agent", 1, "agent", nil), + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent"), + k8sStepRunInnerTests("name=agent-clusterwide-helm-agent", 1, "agent"), + k8sStepRunInnerTests("name=agent-ksmsharded-helm-agent", 1, "agent"), }, }, { name: "helm managed agent default kubernetes privileged", - values: map[string]any{ - "agent": map[string]any{ - "unprivileged": false, - "image": map[string]any{ - "repository": kCtx.agentImageRepo, - "tag": kCtx.agentImageTag, - "pullPolicy": "Never", - }, - "fleet": map[string]any{ - "enabled": true, - "url": kCtx.enrollParams.FleetURL, - "token": kCtx.enrollParams.EnrollmentToken, - "preset": "perNode", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": kCtx.enrollParams.FleetURL, + "token": kCtx.enrollParams.EnrollmentToken, + "preset": "perNode", + }, }, - }, - }, - runK8SInnerTests: true, - // - perNode Daemonset (totalK8SNodes pods) - atLeastAgentPods: totalK8SNodes, - agentPodLabelSelectors: []string{ - // name=agent-{preset}-{release} - "name=agent-pernode-helm-agent", + }), + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil), + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent"), }, }, { name: "helm managed agent default kubernetes unprivileged", - values: map[string]any{ - "agent": map[string]any{ - "unprivileged": true, - "image": map[string]any{ - "repository": kCtx.agentImageRepo, - "tag": kCtx.agentImageTag, - "pullPolicy": "Never", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": true, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": kCtx.enrollParams.FleetURL, + "token": kCtx.enrollParams.EnrollmentToken, + "preset": "perNode", + }, }, - "fleet": map[string]any{ - "enabled": true, - "url": kCtx.enrollParams.FleetURL, - "token": kCtx.enrollParams.EnrollmentToken, - "preset": "perNode", - }, - }, - }, - runK8SInnerTests: true, - // - perNode Daemonset (totalK8SNodes pods) - atLeastAgentPods: totalK8SNodes, - agentPodLabelSelectors: []string{ - // name=agent-{preset}-{release} - "name=agent-pernode-helm-agent", + }), + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil), + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent"), }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if tc.skipReason != "" { + t.Skip(tc.skipReason) + } + ctx := context.Background() testNamespace := kCtx.getNamespace(t) - settings := cli.New() - settings.SetNamespace(testNamespace) - actionConfig := &action.Configuration{} - - helmChart, err := loader.Load(agentK8SHelm) - require.NoError(t, err, "failed to load helm chart") - - err = actionConfig.Init(settings.RESTClientGetter(), settings.Namespace(), "", - func(format string, v ...interface{}) {}) - require.NoError(t, err, "failed to init helm action config") - - helmValues := tc.values - - k8sNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}} - - t.Cleanup(func() { - if t.Failed() { - if err := k8sDumpAllPodLogs(ctx, kCtx.client, testNamespace, testNamespace, kCtx.logsBasePath); err != nil { - t.Logf("failed to dump logs: %s", err) - } - } - - uninstallAction := action.NewUninstall(actionConfig) - uninstallAction.Wait = true - _, err = uninstallAction.Run("helm-agent") - if err != nil { - t.Logf("failed to uninstall helm chart: %s", err) - } - - err = k8sDeleteObjects(ctx, kCtx.client, k8sDeleteOpts{wait: true}, k8sNamespace) - if err != nil { - t.Logf("failed to delete k8s namespace: %s", err) - } - }) - - installAction := action.NewInstall(actionConfig) - installAction.Namespace = testNamespace - installAction.CreateNamespace = true - installAction.UseReleaseName = true - installAction.ReleaseName = "helm-agent" - installAction.Timeout = 2 * time.Minute - installAction.Wait = true - installAction.WaitForJobs = true - _, err = installAction.Run(helmChart, helmValues) - require.NoError(t, err, "failed to install helm chart") - - healthyAgentPods := 0 - for _, podSelector := range tc.agentPodLabelSelectors { - pods := &corev1.PodList{} - err = kCtx.client.Resources(testNamespace).List(ctx, pods, func(opt *metav1.ListOptions) { - opt.LabelSelector = podSelector - }) - require.NoError(t, err, "failed to list pods with selector ", podSelector) - - for _, pod := range pods.Items { - var stdout, stderr bytes.Buffer - err = k8sCheckAgentStatus(ctx, kCtx.client, &stdout, &stderr, testNamespace, pod.Name, "agent", map[string]bool{}) - if err != nil { - t.Errorf("failed to check agent status: %v", err) - t.Logf("stdout: %s\n", stdout.String()) - t.Logf("stderr: %s\n", stderr.String()) - t.FailNow() - } - healthyAgentPods++ - - if !tc.runK8SInnerTests { - continue - } - - stdout.Reset() - stderr.Reset() - err := kCtx.client.Resources().ExecInPod(ctx, testNamespace, pod.Name, "agent", - []string{"/usr/share/elastic-agent/k8s-inner-tests", "-test.v"}, &stdout, &stderr) - t.Logf("%s k8s-inner-tests output:", pod.Name) - t.Log(stdout.String()) - if err != nil { - t.Log(stderr.String()) - } - require.NoError(t, err, "error at k8s inner tests execution") - } + for _, step := range tc.steps { + step(t, ctx, kCtx, testNamespace) } - - require.GreaterOrEqual(t, healthyAgentPods, tc.atLeastAgentPods, - fmt.Sprintf("at least %d agent containers should be checked", tc.atLeastAgentPods)) }) } } -func k8sKustomizeDeployAgent(t *testing.T, ctx context.Context, client klient.Client, objects []k8s.Object, - namespace string, runK8SInnerTests bool, testlogsBasePath string, checkStatus bool, componentPresence map[string]bool, -) { - err := k8sCreateObjects(ctx, client, k8sCreateOpts{namespace: namespace, wait: true}, objects...) - require.NoError(t, err, "failed to create k8s objects") - - t.Cleanup(func() { - if t.Failed() { - if err := k8sDumpAllPodLogs(ctx, client, namespace, namespace, testlogsBasePath); err != nil { - t.Logf("failed to dump logs: %s", err) - } - } - }) - - pods := &corev1.PodList{} - podsLabelSelector := fmt.Sprintf("app=elastic-agent-standalone") - err = client.Resources(namespace).List(ctx, pods, func(opt *metav1.ListOptions) { - opt.LabelSelector = podsLabelSelector - }) - require.NoError(t, err, "failed to list pods with selector ", podsLabelSelector) - require.NotEmpty(t, pods.Items, "no pods found with selector ", podsLabelSelector) - - for _, pod := range pods.Items { - var stdout, stderr bytes.Buffer - - if checkStatus { - err = k8sCheckAgentStatus(ctx, client, &stdout, &stderr, namespace, pod.Name, "elastic-agent-standalone", componentPresence) - if err != nil { - t.Errorf("failed to check agent status: %v", err) - t.Logf("stdout: %s\n", stdout.String()) - t.Logf("stderr: %s\n", stderr.String()) - t.FailNow() - } - } - - stdout.Reset() - stderr.Reset() - - if runK8SInnerTests { - err := client.Resources().ExecInPod(ctx, namespace, pod.Name, "elastic-agent-standalone", - []string{"/usr/share/elastic-agent/k8s-inner-tests", "-test.v"}, &stdout, &stderr) - t.Logf("%s k8s-inner-tests output:", pod.Name) - t.Log(stdout.String()) - if err != nil { - t.Log(stderr.String()) - } - require.NoError(t, err, "error at k8s inner tests execution") - } - } -} - // k8sCheckAgentStatus checks that the agent reports healthy. func k8sCheckAgentStatus(ctx context.Context, client klient.Client, stdout *bytes.Buffer, stderr *bytes.Buffer, namespace string, agentPodName string, containerName string, componentPresence map[string]bool, @@ -806,8 +590,8 @@ func k8sYAMLToObjects(reader *bufio.Reader) ([]k8s.Object, error) { return objects, nil } -// renderKustomize renders the given kustomize directory to YAML -func renderKustomize(kustomizePath string) ([]byte, error) { +// k8sRenderKustomize renders the given kustomize directory to YAML +func k8sRenderKustomize(kustomizePath string) ([]byte, error) { // Create a file system pointing to the kustomize directory fSys := filesys.MakeFsOnDisk() @@ -1072,6 +856,38 @@ func (k8sContext) getNamespace(t *testing.T) string { return noSpecialCharsRegexp.ReplaceAllString(testNamespace, "") } +func k8sSchedulableNodeCount(ctx context.Context, kCtx k8sContext) (int, error) { + nodeList := corev1.NodeList{} + err := kCtx.client.Resources().List(ctx, &nodeList) + if err != nil { + return 0, err + } + + totalSchedulableNodes := 0 + + for _, node := range nodeList.Items { + if node.Spec.Unschedulable { + continue + } + + hasNoScheduleTaint := false + for _, taint := range node.Spec.Taints { + if taint.Effect == corev1.TaintEffectNoSchedule { + hasNoScheduleTaint = true + break + } + } + + if hasNoScheduleTaint { + continue + } + + totalSchedulableNodes++ + } + + return totalSchedulableNodes, err +} + // k8sGetContext performs all the necessary checks to get a k8sContext for the current test func k8sGetContext(t *testing.T, info *define.Info) k8sContext { agentImage := os.Getenv("AGENT_IMAGE") @@ -1118,3 +934,233 @@ func k8sGetContext(t *testing.T, info *define.Info) k8sContext { enrollParams: enrollParams, } } + +// k8sTestStep is a function that performs a single step in a k8s integration test +type k8sTestStep func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) + +// k8sStepCreateNamespace creates a namespace for the current test and adds a test cleanup that +// deletes it +func k8sStepCreateNamespace() k8sTestStep { + return func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + k8sNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + + t.Cleanup(func() { + err := k8sDeleteObjects(ctx, kCtx.client, k8sDeleteOpts{wait: true}, k8sNamespace) + if err != nil { + t.Logf("failed to delete namespace: %v", err) + } + }) + + err := k8sCreateObjects(ctx, kCtx.client, k8sCreateOpts{wait: true}, k8sNamespace) + require.NoError(t, err, "failed to create namespace") + } +} + +// k8sKustomizeOverrides is futile attempt to make kustomize somewhat flexible and +// contains certain handpicked overrides to apply to the k8s objects created from +// kustomize rendering +type k8sKustomizeOverrides struct { + agentContainerRunUser *int64 + agentContainerRunGroup *int64 + agentContainerCapabilitiesDrop []corev1.Capability + agentContainerCapabilitiesAdd []corev1.Capability + agentContainerExtraEnv []corev1.EnvVar + agentContainerArgs []string + agentContainerMemoryLimit string +} + +// k8sStepDeployKustomize renders a kustomize manifest and deploys it. Also, it tries to +// adjust the k8s objects created from the rendering to match the needs of the current test with k8sKustomizeOverrides. +// However, this is not that as flexible as we would like it to be. As a last resort somebody can use forEachObject callback +// to further adjust the k8s objects +func k8sStepDeployKustomize(kustomizePath string, containerName string, overrides k8sKustomizeOverrides, forEachObject func(object k8s.Object)) k8sTestStep { + return func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + renderedManifest, err := k8sRenderKustomize(kustomizePath) + require.NoError(t, err, "failed to render kustomize") + + objects, err := k8sYAMLToObjects(bufio.NewReader(bytes.NewReader(renderedManifest))) + require.NoError(t, err, "failed to parse rendered kustomize") + + if forEachObject != nil { + for _, object := range objects { + forEachObject(object) + } + } + + k8sKustomizeAdjustObjects(objects, namespace, containerName, + func(container *corev1.Container) { + // set agent image + container.Image = kCtx.agentImage + // set ImagePullPolicy to "Never" to avoid pulling the image + // as the image is already loaded by the kubernetes provisioner + container.ImagePullPolicy = "Never" + + if overrides.agentContainerMemoryLimit != "" { + container.Resources.Limits = corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse(overrides.agentContainerMemoryLimit), + } + } + + // if security context overrides are set then set security context + if overrides.agentContainerCapabilitiesDrop != nil || overrides.agentContainerCapabilitiesAdd != nil || + overrides.agentContainerRunUser != nil || overrides.agentContainerRunGroup != nil { + // set security context + container.SecurityContext = &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Drop: overrides.agentContainerCapabilitiesDrop, + Add: overrides.agentContainerCapabilitiesAdd, + }, + RunAsUser: overrides.agentContainerRunUser, + RunAsGroup: overrides.agentContainerRunGroup, + } + } + + // set Elasticsearch host and API key + for idx, env := range container.Env { + if env.Name == "ES_HOST" { + container.Env[idx].Value = kCtx.esHost + container.Env[idx].ValueFrom = nil + } + if env.Name == "API_KEY" { + container.Env[idx].Value = kCtx.esAPIKey + container.Env[idx].ValueFrom = nil + } + } + + if len(overrides.agentContainerExtraEnv) > 0 { + container.Env = append(container.Env, overrides.agentContainerExtraEnv...) + } + + if overrides.agentContainerArgs != nil { + // drop arguments overriding default config + container.Args = []string{} + } + }, + func(pod *corev1.PodSpec) { + for volumeIdx, volume := range pod.Volumes { + // need to update the volume path of the state directory + // to match the test namespace + if volume.Name == "elastic-agent-state" { + hostPathType := corev1.HostPathDirectoryOrCreate + pod.Volumes[volumeIdx].VolumeSource.HostPath = &corev1.HostPathVolumeSource{ + Type: &hostPathType, + Path: fmt.Sprintf("/var/lib/elastic-agent-standalone/%s/state", namespace), + } + } + } + }) + + t.Cleanup(func() { + if t.Failed() { + if err := k8sDumpAllPodLogs(ctx, kCtx.client, namespace, namespace, kCtx.logsBasePath); err != nil { + t.Logf("failed to dump logs: %v", err) + } + } + + err := k8sDeleteObjects(ctx, kCtx.client, k8sDeleteOpts{wait: true}, objects...) + if err != nil { + t.Logf("failed to delete objects: %v", err) + } + }) + + err = k8sCreateObjects(ctx, kCtx.client, k8sCreateOpts{wait: true}, objects...) + require.NoError(t, err, "failed to create objects") + } +} + +// k8sStepCheckAgentStatus checks the status of the agent inside the pods returned by the selector +func k8sStepCheckAgentStatus(agentPodLabelSelector string, expectedPodNumber int, containerName string, componentPresence map[string]bool) k8sTestStep { + return func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + perNodePodList := &corev1.PodList{} + err := kCtx.client.Resources(namespace).List(ctx, perNodePodList, func(opt *metav1.ListOptions) { + opt.LabelSelector = agentPodLabelSelector + }) + require.NoError(t, err, "failed to list pods with selector ", perNodePodList) + require.NotEmpty(t, perNodePodList.Items, "no pods found with selector ", perNodePodList) + require.Equal(t, expectedPodNumber, len(perNodePodList.Items), "unexpected number of pods found with selector ", perNodePodList) + + for _, pod := range perNodePodList.Items { + var stdout, stderr bytes.Buffer + err = k8sCheckAgentStatus(ctx, kCtx.client, &stdout, &stderr, namespace, pod.Name, containerName, componentPresence) + if err != nil { + t.Errorf("failed to check agent status %s: %v", pod.Name, err) + t.Logf("stdout: %s\n", stdout.String()) + t.Logf("stderr: %s\n", stderr.String()) + t.FailNow() + } + } + } +} + +// k8sStepRunInnerTests invokes the k8s inner tests inside the pods returned by the selector. Note that this +// step requires the agent image to be built with the testing framework as there is the point where the binary +// for the inner tests is copied +func k8sStepRunInnerTests(agentPodLabelSelector string, expectedPodNumber int, containerName string) k8sTestStep { + return func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + perNodePodList := &corev1.PodList{} + err := kCtx.client.Resources(namespace).List(ctx, perNodePodList, func(opt *metav1.ListOptions) { + opt.LabelSelector = agentPodLabelSelector + }) + require.NoError(t, err, "failed to list pods with selector ", perNodePodList) + require.NotEmpty(t, perNodePodList.Items, "no pods found with selector ", perNodePodList) + require.Equal(t, expectedPodNumber, len(perNodePodList.Items), "unexpected number of pods found with selector ", perNodePodList) + + for _, pod := range perNodePodList.Items { + var stdout, stderr bytes.Buffer + err = kCtx.client.Resources().ExecInPod(ctx, namespace, pod.Name, containerName, + []string{"/usr/share/elastic-agent/k8s-inner-tests", "-test.v"}, &stdout, &stderr) + t.Logf("%s k8s-inner-tests output:", pod.Name) + t.Log(stdout.String()) + if err != nil { + t.Log(stderr.String()) + } + require.NoError(t, err, "error at k8s inner tests execution") + } + } +} + +// k8sStepHelmDeploy deploys a helm chart with the given values and the release name +func k8sStepHelmDeploy(chartPath string, releaseName string, values map[string]any) k8sTestStep { + return func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + settings := cli.New() + settings.SetNamespace(namespace) + actionConfig := &action.Configuration{} + + helmChart, err := loader.Load(chartPath) + require.NoError(t, err, "failed to load helm chart") + + err = actionConfig.Init(settings.RESTClientGetter(), settings.Namespace(), "", + func(format string, v ...interface{}) {}) + require.NoError(t, err, "failed to init helm action config") + + t.Cleanup(func() { + if t.Failed() { + if err := k8sDumpAllPodLogs(ctx, kCtx.client, namespace, namespace, kCtx.logsBasePath); err != nil { + t.Logf("failed to dump logs: %v", err) + } + } + + uninstallAction := action.NewUninstall(actionConfig) + uninstallAction.Wait = true + _, err = uninstallAction.Run(releaseName) + if err != nil { + t.Logf("failed to uninstall helm chart: %v", err) + } + }) + + installAction := action.NewInstall(actionConfig) + installAction.Namespace = namespace + installAction.CreateNamespace = true + installAction.UseReleaseName = true + installAction.ReleaseName = releaseName + installAction.Timeout = 2 * time.Minute + installAction.Wait = true + installAction.WaitForJobs = true + _, err = installAction.Run(helmChart, values) + require.NoError(t, err, "failed to install helm chart") + } +}