From 3c17bc8850a89a01f14bb1b46a1d9855023e868c Mon Sep 17 00:00:00 2001 From: kaanyalti Date: Wed, 4 Dec 2024 08:38:25 -0500 Subject: [PATCH] enhancement(5832): added kubernetes test --- .../kubernetes_agent_standalone_test.go | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/kubernetes_agent_standalone_test.go index 86db561edd8..36c91942b8d 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/kubernetes_agent_standalone_test.go @@ -48,6 +48,7 @@ import ( "helm.sh/helm/v3/pkg/cli" helmKube "helm.sh/helm/v3/pkg/kube" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator" aclient "github.com/elastic/elastic-agent/pkg/control/v2/client" atesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" @@ -556,6 +557,215 @@ func TestKubernetesAgentHelm(t *testing.T) { } } +func TestRestrictCliUpgrade(t *testing.T) { + info := define.Require(t, define.Requirements{ + Stack: &define.Stack{}, + Local: false, + Sudo: false, + OS: []define.OS{ + {Type: define.Kubernetes, DockerVariant: "basic"}, + {Type: define.Kubernetes, DockerVariant: "wolfi"}, + {Type: define.Kubernetes, DockerVariant: "ubi"}, + {Type: define.Kubernetes, DockerVariant: "complete"}, + {Type: define.Kubernetes, DockerVariant: "complete-wolfi"}, + }, + Group: define.Kubernetes, + }) + + agentImage := os.Getenv("AGENT_IMAGE") + require.NotEmpty(t, agentImage, "AGENT_IMAGE must be set") + + agentImageParts := strings.SplitN(agentImage, ":", 2) + require.Len(t, agentImageParts, 2, "AGENT_IMAGE must be in the form ':'") + agentImageRepo := agentImageParts[0] + agentImageTag := agentImageParts[1] + + client, err := info.KubeClient() + require.NoError(t, err) + require.NotNil(t, client) + + testLogsBasePath := os.Getenv("K8S_TESTS_POD_LOGS_BASE") + require.NotEmpty(t, testLogsBasePath, "K8S_TESTS_POD_LOGS_BASE must be set") + + err = os.MkdirAll(filepath.Join(testLogsBasePath, t.Name()), 0755) + require.NoError(t, err, "failed to create test logs directory") + + namespace := info.Namespace + + esHost := os.Getenv("ELASTICSEARCH_HOST") + require.NotEmpty(t, esHost, "ELASTICSEARCH_HOST must be set") + + esAPIKey, err := generateESAPIKey(info.ESClient, namespace) + require.NoError(t, err, "failed to generate ES API key") + require.NotEmpty(t, esAPIKey, "failed to generate ES API key") + + require.NoError(t, err, "failed to create fleet enroll params") + + testCases := []struct { + name string + values map[string]any + atLeastValidatedPodsNumber int + runK8SInnerTests bool + }{ + { + name: "helm standalone agent default kubernetes privileged", + values: map[string]any{ + "kubernetes": map[string]any{ + "enabled": true, + }, + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": agentImageRepo, + "tag": agentImageTag, + "pullPolicy": "Never", + }, + }, + "outputs": map[string]any{ + "default": map[string]any{ + "type": "ESPlainAuthAPI", + "url": esHost, + "api_key": esAPIKey, + }, + }, + }, + runK8SInnerTests: true, + // - perNode Daemonset (at least 1 agent pod) + // - clusterWide Deployment (1 agent pod) + // - ksmSharded Statefulset (1 agent pod) + atLeastValidatedPodsNumber: 3, + }, + { + name: "helm standalone agent default kubernetes unprivileged", + values: map[string]any{ + "kubernetes": map[string]any{ + "enabled": true, + }, + "agent": map[string]any{ + "unprivileged": true, + "image": map[string]any{ + "repository": agentImageRepo, + "tag": agentImageTag, + "pullPolicy": "Never", + }, + }, + "outputs": map[string]any{ + "default": map[string]any{ + "type": "ESPlainAuthAPI", + "url": esHost, + "api_key": esAPIKey, + }, + }, + }, + runK8SInnerTests: true, + // - perNode Daemonset (at least 1 agent pod) + // - clusterWide Deployment (1 agent pod) + // - ksmSharded Statefulset (1 agent pod) + atLeastValidatedPodsNumber: 3, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + hasher := sha256.New() + hasher.Write([]byte(tc.name)) + testNamespace := strings.ToLower(base64.URLEncoding.EncodeToString(hasher.Sum(nil))) + testNamespace = noSpecialCharsRegexp.ReplaceAllString(testNamespace, "") + + settings := cli.New() + settings.SetNamespace(testNamespace) + actionConfig := &action.Configuration{} + + helmChart, err := loader.Load(agentK8SHelm) + require.NoError(t, err, "failed to load helm chart") + + err = actionConfig.Init(settings.RESTClientGetter(), settings.Namespace(), "", + func(format string, v ...interface{}) {}) + require.NoError(t, err, "failed to init helm action config") + + helmValues := tc.values + + t.Cleanup(func() { + if t.Failed() { + k8sDumpAllPodLogs(ctx, client, testNamespace, testNamespace, testLogsBasePath) + } + + uninstallAction := action.NewUninstall(actionConfig) + uninstallAction.Wait = true + + _, err = uninstallAction.Run("helm-agent") + if err != nil { + require.NoError(t, err, "failed to uninstall helm chart") + } + }) + + installAction := action.NewInstall(actionConfig) + installAction.Namespace = testNamespace + installAction.CreateNamespace = true + installAction.UseReleaseName = true + installAction.ReleaseName = "helm-agent" + installAction.Timeout = 2 * time.Minute + installAction.Wait = true + installAction.WaitForJobs = true + _, err = installAction.Run(helmChart, helmValues) + require.NoError(t, err, "failed to install helm chart") + + podList := &corev1.PodList{} + err = client.Resources(testNamespace).List(ctx, podList) + require.NoError(t, err, fmt.Sprintf("failed to list pods in namespace %s", testNamespace)) + + checkedAgentContainers := 0 + + for _, pod := range podList.Items { + if !strings.HasPrefix(pod.GetName(), "agent-") { + continue + } + + command := []string{"elastic-agent", "status"} + var stdout, stderr bytes.Buffer + var agentHealthyErr error + // we will wait maximum 120 seconds for the agent to report healthy + for i := 0; i < 120; i++ { + stdout.Reset() + stderr.Reset() + agentHealthyErr = client.Resources().ExecInPod(ctx, testNamespace, pod.Name, "agent", command, &stdout, &stderr) + if agentHealthyErr == nil { + break + } + time.Sleep(time.Second * 1) + } + + statusString := stdout.String() + if agentHealthyErr != nil { + t.Errorf("elastic-agent never reported healthy: %v", agentHealthyErr) + t.Logf("stdout: %s\n", statusString) + t.Logf("stderr: %s\n", stderr.String()) + t.FailNow() + return + } + + stdout.Reset() + stderr.Reset() + + upgradeCmd := []string{"elastic-agent", "upgrade", "1.0.0"} + upgradeCmdError := client.Resources().ExecInPod(ctx, testNamespace, pod.Name, "agent", upgradeCmd, &stdout, &stderr) + + errOut := stderr.String() + require.Error(t, upgradeCmdError) + require.Contains(t, errOut, coordinator.ErrNotUpgradable.Error()) + + stderr.Reset() + + checkedAgentContainers++ + } + + require.GreaterOrEqual(t, checkedAgentContainers, tc.atLeastValidatedPodsNumber, + fmt.Sprintf("at least %d agent containers should be checked", tc.atLeastValidatedPodsNumber)) + }) + } +} + func k8sKustomizeDeployAgent(t *testing.T, ctx context.Context, client klient.Client, objects []k8s.Object, namespace string, runK8SInnerTests bool, testlogsBasePath string, checkStatus bool, componentPresence map[string]bool, ) {