From 4a26977d71753e0c0c429d43abbb07fb811bb558 Mon Sep 17 00:00:00 2001 From: Christian Zunker Date: Mon, 20 Nov 2023 06:00:19 +0100 Subject: [PATCH] Add e2e test Signed-off-by: Christian Zunker --- .../operator/operator_status.go | 64 ++++++++-------- pkg/utils/k8s/pods.go | 23 ++++++ tests/integration/audit_config_base_suite.go | 39 +++++++++- .../audit_config_namespace_test.go | 72 +++++++++--------- tests/integration/audit_config_test.go | 75 ++++++++++--------- .../integration/audit_config_upgrade_test.go | 7 +- 6 files changed, 173 insertions(+), 107 deletions(-) create mode 100644 pkg/utils/k8s/pods.go diff --git a/cmd/mondoo-operator/operator/operator_status.go b/cmd/mondoo-operator/operator/operator_status.go index c5ddba19c..dab9118b4 100644 --- a/cmd/mondoo-operator/operator/operator_status.go +++ b/cmd/mondoo-operator/operator/operator_status.go @@ -12,6 +12,7 @@ import ( k8sv1alpha2 "go.mondoo.com/mondoo-operator/api/v1alpha2" "go.mondoo.com/mondoo-operator/controllers" "go.mondoo.com/mondoo-operator/controllers/status" + "go.mondoo.com/mondoo-operator/pkg/utils/k8s" "go.mondoo.com/mondoo-operator/pkg/utils/mondoo" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" @@ -54,35 +55,36 @@ func checkForTerminatedState(ctx context.Context, nonCacheClient client.Client, logger.Error(err, "failed to list pods", "Mondoo.Namespace", mondooAuditConfig.Namespace, "Mondoo.Name", mondooAuditConfig.Name) return err } - for _, pod := range podList.Items { - for _, containerStatus := range pod.Status.ContainerStatuses { - if containerStatus.Name != "manager" { - continue - } - stateUpdate := false - if containerStatus.LastTerminationState.Terminated != nil && containerStatus.LastTerminationState.Terminated.ExitCode != 0 { - logger.Info("mondoo-operator was terminated before") - // Update status - updateOperatorConditions(&mondooAuditConfig, true, &pod) - stateUpdate = true - } else if containerStatus.State.Running != nil { - updateOperatorConditions(&mondooAuditConfig, false, &corev1.Pod{}) - stateUpdate = true + + currentPod := k8s.GetNewestPodFromList(podList) + for _, containerStatus := range currentPod.Status.ContainerStatuses { + if containerStatus.Name != "manager" { + continue + } + stateUpdate := false + if containerStatus.State.Terminated != nil || containerStatus.LastTerminationState.Terminated != nil { + logger.Info("mondoo-operator was terminated before") + // Update status + updateOperatorConditions(&mondooAuditConfig, true, currentPod) + stateUpdate = true + } else if containerStatus.RestartCount == 0 && containerStatus.State.Terminated == nil { + logger.Info("mondoo-operator is running or starting", "state", containerStatus.State) + updateOperatorConditions(&mondooAuditConfig, false, &corev1.Pod{}) + stateUpdate = true + } + if stateUpdate { + err := mondoo.UpdateMondooAuditStatus(ctx, nonCacheClient, mondooAuditConfigCopy, &mondooAuditConfig, logger) + if err != nil { + logger.Error(err, "failed to update status for MondooAuditConfig") + return err } - if stateUpdate { - err := mondoo.UpdateMondooAuditStatus(ctx, nonCacheClient, mondooAuditConfigCopy, &mondooAuditConfig, logger) - if err != nil { - logger.Error(err, "failed to update status for MondooAuditConfig") - return err - } - // Report upstream before we get OOMkilled again - err = statusReport.Report(ctx, mondooAuditConfig, *config) - if err != nil { - logger.Error(err, "failed to report status upstream") - return err - } - break + // Report upstream before we get OOMkilled again + err = statusReport.Report(ctx, mondooAuditConfig, *config) + if err != nil { + logger.Error(err, "failed to report status upstream") + return err } + break } } } @@ -98,12 +100,12 @@ func updateOperatorConditions(config *k8sv1alpha2.MondooAuditConfig, degradedSta memoryLimit := "" if degradedStatus { msg = "Mondoo Operator controller is unavailable" - for _, status := range pod.Status.ContainerStatuses { - if status.LastTerminationState.Terminated != nil && status.LastTerminationState.Terminated.ExitCode == 137 { - // TODO: double check container name? + for i, containerStatus := range pod.Status.ContainerStatuses { + if (containerStatus.LastTerminationState.Terminated != nil && containerStatus.LastTerminationState.Terminated.ExitCode == 137) || + (containerStatus.State.Terminated != nil && containerStatus.State.Terminated.ExitCode == 137) { msg = "Mondoo Operator controller is unavailable due to OOM" affectedPods = append(affectedPods, pod.Name) - memoryLimit = pod.Spec.Containers[0].Resources.Limits.Memory().String() + memoryLimit = pod.Spec.Containers[i].Resources.Limits.Memory().String() break } } diff --git a/pkg/utils/k8s/pods.go b/pkg/utils/k8s/pods.go new file mode 100644 index 000000000..0be282d81 --- /dev/null +++ b/pkg/utils/k8s/pods.go @@ -0,0 +1,23 @@ +package k8s + +import ( + "time" + + corev1 "k8s.io/api/core/v1" +) + +// GetNewestPodFromList returns the most recent pod from a pod list +// This is determined by the creation timestamp of the pod +func GetNewestPodFromList(pods *corev1.PodList) *corev1.Pod { + podCreationtime := time.Unix(0, 0) + currentPod := &corev1.Pod{} + for i := range pods.Items { + pod := &pods.Items[i] + if pod.ObjectMeta.CreationTimestamp.Time.Before(podCreationtime) { + continue + } + podCreationtime = pod.ObjectMeta.CreationTimestamp.Time + currentPod = pod + } + return currentPod +} diff --git a/tests/integration/audit_config_base_suite.go b/tests/integration/audit_config_base_suite.go index b68800bc7..41acf3d5c 100644 --- a/tests/integration/audit_config_base_suite.go +++ b/tests/integration/audit_config_base_suite.go @@ -247,17 +247,52 @@ func (s *AuditConfigBaseSuite) testOOMMondooOperatorController(auditConfig mondo corev1.ResourceMemory: resource.MustParse("15Mi"), // this should be low enough to trigger an OOMkilled } - s.NoError(s.testCluster.K8sHelper.Clientset.Update(s.ctx, &operatorDeployment, nil)) + zap.S().Info("Reducing memory limit to trigger OOM.") + s.NoError(s.testCluster.K8sHelper.Clientset.Update(s.ctx, &operatorDeployment)) // This will take some time, because: // a new replicaset should be created // the first Pod tries to start and gets killed // on the 2nd start we should get an OOMkilled status update - s.testCluster.K8sHelper.CheckForDegradedCondition(&auditConfig, mondoov2.MondooOperaotrDegraded, corev1.ConditionTrue) + err := s.testCluster.K8sHelper.CheckForDegradedCondition(&auditConfig, mondoov2.MondooOperaotrDegraded, corev1.ConditionTrue) + s.NoError(err, "Failed to find degraded condition") + + foundMondooAuditConfig, err := s.testCluster.K8sHelper.GetMondooAuditConfigFromCluster(auditConfig.Name, auditConfig.Namespace) + s.NoError(err, "Failed to find MondooAuditConfig") + s.Contains(foundMondooAuditConfig.Status.Conditions[5].Message, "OOM", "Failed to find OOMKilled message in degraded condition") + s.Len(foundMondooAuditConfig.Status.Conditions[5].AffectedPods, 1, "Failed to find only one pod in degraded condition") + + // Give the integration a chance to update + time.Sleep(2 * time.Second) status, err := s.integration.GetStatus(s.ctx) s.NoError(err, "Failed to get status") s.Equal("ERROR", status) + + s.NoError(s.testCluster.K8sHelper.Clientset.List(s.ctx, deployments, listOpts)) + s.Equalf(1, len(deployments.Items), "mondoo-operator deployment not found") + + operatorDeployment = deployments.Items[0] + operatorDeployment.Spec.Template.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("100Mi"), // this should be enough to get the operator running again + } + + zap.S().Info("Increasing memory limit to get controller running again.") + s.NoError(s.testCluster.K8sHelper.Clientset.Update(s.ctx, &operatorDeployment)) + + err = s.testCluster.K8sHelper.CheckForDegradedCondition(&auditConfig, mondoov2.MondooOperaotrDegraded, corev1.ConditionFalse) + s.NoError(err, "Failed to find degraded condition") + foundMondooAuditConfig, err = s.testCluster.K8sHelper.GetMondooAuditConfigFromCluster(auditConfig.Name, auditConfig.Namespace) + s.NoError(err, "Failed to find MondooAuditConfig") + s.NotContains(foundMondooAuditConfig.Status.Conditions[5].Message, "OOM", "Found OOMKilled message in condition") + s.Len(foundMondooAuditConfig.Status.Conditions[5].AffectedPods, 0, "Found a pod in condition") + + // Give the integration a chance to update + time.Sleep(2 * time.Second) + + status, err = s.integration.GetStatus(s.ctx) + s.NoError(err, "Failed to get status") + s.Equal("ACTIVE", status) } func (s *AuditConfigBaseSuite) testMondooAuditConfigContainers(auditConfig mondoov2.MondooAuditConfig) { diff --git a/tests/integration/audit_config_namespace_test.go b/tests/integration/audit_config_namespace_test.go index bade8adf3..b73e8e45f 100644 --- a/tests/integration/audit_config_namespace_test.go +++ b/tests/integration/audit_config_namespace_test.go @@ -4,6 +4,10 @@ package integration import ( + "testing" + + "github.com/stretchr/testify/suite" + "go.mondoo.com/mondoo-operator/tests/framework/utils" "go.uber.org/zap" "sigs.k8s.io/controller-runtime/pkg/client" @@ -74,44 +78,44 @@ func (s *AuditConfigCustomNamespaceSuite) TearDownSuite() { s.AuditConfigBaseSuite.TearDownSuite() } -/* - func (s *AuditConfigCustomNamespaceSuite) TestReconcile_KubernetesResources() { - auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, true, false, false, false) - auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name - s.testMondooAuditConfigKubernetesResources(auditConfig) - } +func (s *AuditConfigCustomNamespaceSuite) TestReconcile_KubernetesResources() { + auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, true, false, false, false) + auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name + s.testMondooAuditConfigKubernetesResources(auditConfig) +} - func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Containers() { - auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, true, false, false) - auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name +func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Containers() { + auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, true, false, false) + auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name - // Ignore the operator namespace and the scanner namespace because we cannot scan a local image - // Ignore kube-system to speed up the containers test - auditConfig.Spec.Filtering.Namespaces.Exclude = []string{s.ns.Name, s.testCluster.Settings.Namespace, "kube-system"} - s.testMondooAuditConfigContainers(auditConfig) - } + // Ignore the operator namespace and the scanner namespace because we cannot scan a local image + // Ignore kube-system to speed up the containers test + auditConfig.Spec.Filtering.Namespaces.Exclude = []string{s.ns.Name, s.testCluster.Settings.Namespace, "kube-system"} + s.testMondooAuditConfigContainers(auditConfig) +} - func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Nodes() { - auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, true, false) - auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name - s.testMondooAuditConfigNodes(auditConfig) - } +func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Nodes() { + auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, true, false) + auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name + s.testMondooAuditConfigNodes(auditConfig) +} - func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Admission() { - auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, false, true) - auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name - auditConfig.Spec.Admission.ServiceAccountName = s.webhookServiceAccount.Name - s.testMondooAuditConfigAdmission(auditConfig) - } +func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Admission() { + auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, false, true) + auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name + auditConfig.Spec.Admission.ServiceAccountName = s.webhookServiceAccount.Name + s.testMondooAuditConfigAdmission(auditConfig) +} - func (s *AuditConfigCustomNamespaceSuite) TestReconcile_AdmissionMissingSA() { - auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, false, true) - auditConfig.Spec.Scanner.ServiceAccountName = "missing-serviceaccount" - auditConfig.Spec.Admission.ServiceAccountName = s.webhookServiceAccount.Name - s.testMondooAuditConfigAdmissionMissingSA(auditConfig) - } - func TestAuditConfigCustomNamespaceSuite(t *testing.T) { - s := new(AuditConfigCustomNamespaceSuite) +func (s *AuditConfigCustomNamespaceSuite) TestReconcile_AdmissionMissingSA() { + auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, false, true) + auditConfig.Spec.Scanner.ServiceAccountName = "missing-serviceaccount" + auditConfig.Spec.Admission.ServiceAccountName = s.webhookServiceAccount.Name + s.testMondooAuditConfigAdmissionMissingSA(auditConfig) +} + +func TestAuditConfigCustomNamespaceSuite(t *testing.T) { + s := new(AuditConfigCustomNamespaceSuite) defer func(s *AuditConfigCustomNamespaceSuite) { HandlePanics(recover(), func() { if err := s.testCluster.UninstallOperator(); err != nil { @@ -126,5 +130,3 @@ func (s *AuditConfigCustomNamespaceSuite) TearDownSuite() { }(s) suite.Run(t, s) } - -*/ diff --git a/tests/integration/audit_config_test.go b/tests/integration/audit_config_test.go index b6c5091b8..0291c291c 100644 --- a/tests/integration/audit_config_test.go +++ b/tests/integration/audit_config_test.go @@ -7,8 +7,10 @@ import ( "testing" "github.com/stretchr/testify/suite" + "go.mondoo.com/mondoo-operator/api/v1alpha2" "go.mondoo.com/mondoo-operator/tests/framework/utils" "go.uber.org/zap" + "k8s.io/utils/ptr" ) type AuditConfigSuite struct { @@ -20,50 +22,49 @@ func (s *AuditConfigSuite) TestOOMControllerReporting() { s.testOOMMondooOperatorController(auditConfig) } -/* - func (s *AuditConfigSuite) TestReconcile_AllDisabled() { - auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, false) - s.testMondooAuditConfigAllDisabled(auditConfig) - } +func (s *AuditConfigSuite) TestReconcile_AllDisabled() { + auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, false) + s.testMondooAuditConfigAllDisabled(auditConfig) +} - func (s *AuditConfigSuite) TestReconcile_KubernetesResources() { - auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, true, false, false, false) - s.testMondooAuditConfigKubernetesResources(auditConfig) - } +func (s *AuditConfigSuite) TestReconcile_KubernetesResources() { + auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, true, false, false, false) + s.testMondooAuditConfigKubernetesResources(auditConfig) +} - func (s *AuditConfigSuite) TestReconcile_Containers() { - auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, true, false, false) +func (s *AuditConfigSuite) TestReconcile_Containers() { + auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, true, false, false) - // Ignore the operator namespace because we cannot scan a local image - // Ignore kube-system to speed up the containers test - auditConfig.Spec.Filtering.Namespaces.Exclude = []string{s.testCluster.Settings.Namespace, "kube-system"} - s.testMondooAuditConfigContainers(auditConfig) - } + // Ignore the operator namespace because we cannot scan a local image + // Ignore kube-system to speed up the containers test + auditConfig.Spec.Filtering.Namespaces.Exclude = []string{s.testCluster.Settings.Namespace, "kube-system"} + s.testMondooAuditConfigContainers(auditConfig) +} - func (s *AuditConfigSuite) TestReconcile_Nodes() { - auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, true, false) - s.testMondooAuditConfigNodes(auditConfig) - } +func (s *AuditConfigSuite) TestReconcile_Nodes() { + auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, true, false) + s.testMondooAuditConfigNodes(auditConfig) +} - func (s *AuditConfigSuite) TestReconcile_AdmissionPermissive() { - auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true) - s.testMondooAuditConfigAdmission(auditConfig) - } +func (s *AuditConfigSuite) TestReconcile_AdmissionPermissive() { + auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true) + s.testMondooAuditConfigAdmission(auditConfig) +} + +func (s *AuditConfigSuite) TestReconcile_AdmissionEnforcing() { + auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true) + auditConfig.Spec.Admission.Mode = v1alpha2.Enforcing + s.testMondooAuditConfigAdmission(auditConfig) +} - func (s *AuditConfigSuite) TestReconcile_AdmissionEnforcing() { - auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true) - auditConfig.Spec.Admission.Mode = v1alpha2.Enforcing - s.testMondooAuditConfigAdmission(auditConfig) - } +func (s *AuditConfigSuite) TestReconcile_AdmissionEnforcingScaleDownScanApi() { + auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true) + auditConfig.Spec.Admission.Mode = v1alpha2.Enforcing + auditConfig.Spec.Admission.Replicas = ptr.To(int32(1)) + auditConfig.Spec.Scanner.Replicas = ptr.To(int32(1)) + s.testMondooAuditConfigAdmissionScaleDownScanApi(auditConfig) +} - func (s *AuditConfigSuite) TestReconcile_AdmissionEnforcingScaleDownScanApi() { - auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true) - auditConfig.Spec.Admission.Mode = v1alpha2.Enforcing - auditConfig.Spec.Admission.Replicas = ptr.To(int32(1)) - auditConfig.Spec.Scanner.Replicas = ptr.To(int32(1)) - s.testMondooAuditConfigAdmissionScaleDownScanApi(auditConfig) - } -*/ func TestAuditConfigSuite(t *testing.T) { s := new(AuditConfigSuite) defer func(s *AuditConfigSuite) { diff --git a/tests/integration/audit_config_upgrade_test.go b/tests/integration/audit_config_upgrade_test.go index 6b54d5754..bcf562472 100644 --- a/tests/integration/audit_config_upgrade_test.go +++ b/tests/integration/audit_config_upgrade_test.go @@ -4,7 +4,12 @@ package integration import ( + "testing" + + "github.com/stretchr/testify/suite" "go.mondoo.com/mondoo-operator/tests/framework/installer" + "go.mondoo.com/mondoo-operator/tests/framework/utils" + "go.uber.org/zap" ) type AuditConfigUpgradeSuite struct { @@ -22,7 +27,6 @@ func (s *AuditConfigUpgradeSuite) TearDownSuite() { s.NoError(s.spaceClient.Delete(s.ctx)) } -/* func (s *AuditConfigUpgradeSuite) TestUpgradePreviousReleaseToLatest() { auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, true, false, true, false) s.testUpgradePreviousReleaseToLatest(auditConfig) @@ -45,4 +49,3 @@ func TestAuditConfigUpgradeSuite(t *testing.T) { }(s) suite.Run(t, s) } -*/