Skip to content

Commit

Permalink
Add e2e test
Browse files Browse the repository at this point in the history 
Signed-off-by: Christian Zunker <[email protected]>
  • Loading branch information
@czunker
czunker committed Nov 20, 2023
1 parent 4b57a18 commit 4a26977
Show file tree
Hide file tree
Showing 6 changed files with 173 additions and 107 deletions.
64 changes: 33 additions & 31 deletions cmd/mondoo-operator/operator/operator_status.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
k8sv1alpha2 "go.mondoo.com/mondoo-operator/api/v1alpha2"
"go.mondoo.com/mondoo-operator/controllers"
"go.mondoo.com/mondoo-operator/controllers/status"
"go.mondoo.com/mondoo-operator/pkg/utils/k8s"
"go.mondoo.com/mondoo-operator/pkg/utils/mondoo"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -54,35 +55,36 @@ func checkForTerminatedState(ctx context.Context, nonCacheClient client.Client,
logger.Error(err, "failed to list pods", "Mondoo.Namespace", mondooAuditConfig.Namespace, "Mondoo.Name", mondooAuditConfig.Name)
return err
}
for _, pod := range podList.Items {
for _, containerStatus := range pod.Status.ContainerStatuses {
if containerStatus.Name != "manager" {
continue
}
stateUpdate := false
if containerStatus.LastTerminationState.Terminated != nil && containerStatus.LastTerminationState.Terminated.ExitCode != 0 {
logger.Info("mondoo-operator was terminated before")
// Update status
updateOperatorConditions(&mondooAuditConfig, true, &pod)
stateUpdate = true
} else if containerStatus.State.Running != nil {
updateOperatorConditions(&mondooAuditConfig, false, &corev1.Pod{})
stateUpdate = true

currentPod := k8s.GetNewestPodFromList(podList)
for _, containerStatus := range currentPod.Status.ContainerStatuses {
if containerStatus.Name != "manager" {
continue
}
stateUpdate := false
if containerStatus.State.Terminated != nil || containerStatus.LastTerminationState.Terminated != nil {
logger.Info("mondoo-operator was terminated before")
// Update status
updateOperatorConditions(&mondooAuditConfig, true, currentPod)
stateUpdate = true
} else if containerStatus.RestartCount == 0 && containerStatus.State.Terminated == nil {
logger.Info("mondoo-operator is running or starting", "state", containerStatus.State)
updateOperatorConditions(&mondooAuditConfig, false, &corev1.Pod{})
stateUpdate = true
}
if stateUpdate {
err := mondoo.UpdateMondooAuditStatus(ctx, nonCacheClient, mondooAuditConfigCopy, &mondooAuditConfig, logger)
if err != nil {
logger.Error(err, "failed to update status for MondooAuditConfig")
return err
}
if stateUpdate {
err := mondoo.UpdateMondooAuditStatus(ctx, nonCacheClient, mondooAuditConfigCopy, &mondooAuditConfig, logger)
if err != nil {
logger.Error(err, "failed to update status for MondooAuditConfig")
return err
}
// Report upstream before we get OOMkilled again
err = statusReport.Report(ctx, mondooAuditConfig, *config)
if err != nil {
logger.Error(err, "failed to report status upstream")
return err
}
break
// Report upstream before we get OOMkilled again
err = statusReport.Report(ctx, mondooAuditConfig, *config)
if err != nil {
logger.Error(err, "failed to report status upstream")
return err
}
break
}
}
}
Expand All @@ -98,12 +100,12 @@ func updateOperatorConditions(config *k8sv1alpha2.MondooAuditConfig, degradedSta
memoryLimit := ""
if degradedStatus {
msg = "Mondoo Operator controller is unavailable"
for _, status := range pod.Status.ContainerStatuses {
if status.LastTerminationState.Terminated != nil && status.LastTerminationState.Terminated.ExitCode == 137 {
// TODO: double check container name?
for i, containerStatus := range pod.Status.ContainerStatuses {
if (containerStatus.LastTerminationState.Terminated != nil && containerStatus.LastTerminationState.Terminated.ExitCode == 137) ||
(containerStatus.State.Terminated != nil && containerStatus.State.Terminated.ExitCode == 137) {
msg = "Mondoo Operator controller is unavailable due to OOM"
affectedPods = append(affectedPods, pod.Name)
memoryLimit = pod.Spec.Containers[0].Resources.Limits.Memory().String()
memoryLimit = pod.Spec.Containers[i].Resources.Limits.Memory().String()
break
}
}
Expand Down
23 changes: 23 additions & 0 deletions pkg/utils/k8s/pods.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package k8s

import (
"time"

corev1 "k8s.io/api/core/v1"
)

// GetNewestPodFromList returns the most recent pod from a pod list
// This is determined by the creation timestamp of the pod
func GetNewestPodFromList(pods *corev1.PodList) *corev1.Pod {
podCreationtime := time.Unix(0, 0)
currentPod := &corev1.Pod{}
for i := range pods.Items {
pod := &pods.Items[i]
if pod.ObjectMeta.CreationTimestamp.Time.Before(podCreationtime) {
continue
}
podCreationtime = pod.ObjectMeta.CreationTimestamp.Time
currentPod = pod
}
return currentPod
}
39 changes: 37 additions & 2 deletions tests/integration/audit_config_base_suite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,17 +247,52 @@ func (s *AuditConfigBaseSuite) testOOMMondooOperatorController(auditConfig mondo
corev1.ResourceMemory: resource.MustParse("15Mi"), // this should be low enough to trigger an OOMkilled
}

s.NoError(s.testCluster.K8sHelper.Clientset.Update(s.ctx, &operatorDeployment, nil))
zap.S().Info("Reducing memory limit to trigger OOM.")
s.NoError(s.testCluster.K8sHelper.Clientset.Update(s.ctx, &operatorDeployment))

// This will take some time, because:
// a new replicaset should be created
// the first Pod tries to start and gets killed
// on the 2nd start we should get an OOMkilled status update
s.testCluster.K8sHelper.CheckForDegradedCondition(&auditConfig, mondoov2.MondooOperaotrDegraded, corev1.ConditionTrue)
err := s.testCluster.K8sHelper.CheckForDegradedCondition(&auditConfig, mondoov2.MondooOperaotrDegraded, corev1.ConditionTrue)
s.NoError(err, "Failed to find degraded condition")

foundMondooAuditConfig, err := s.testCluster.K8sHelper.GetMondooAuditConfigFromCluster(auditConfig.Name, auditConfig.Namespace)
s.NoError(err, "Failed to find MondooAuditConfig")
s.Contains(foundMondooAuditConfig.Status.Conditions[5].Message, "OOM", "Failed to find OOMKilled message in degraded condition")
s.Len(foundMondooAuditConfig.Status.Conditions[5].AffectedPods, 1, "Failed to find only one pod in degraded condition")

// Give the integration a chance to update
time.Sleep(2 * time.Second)

status, err := s.integration.GetStatus(s.ctx)
s.NoError(err, "Failed to get status")
s.Equal("ERROR", status)

s.NoError(s.testCluster.K8sHelper.Clientset.List(s.ctx, deployments, listOpts))
s.Equalf(1, len(deployments.Items), "mondoo-operator deployment not found")

operatorDeployment = deployments.Items[0]
operatorDeployment.Spec.Template.Spec.Containers[0].Resources.Limits = corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("100Mi"), // this should be enough to get the operator running again
}

zap.S().Info("Increasing memory limit to get controller running again.")
s.NoError(s.testCluster.K8sHelper.Clientset.Update(s.ctx, &operatorDeployment))

err = s.testCluster.K8sHelper.CheckForDegradedCondition(&auditConfig, mondoov2.MondooOperaotrDegraded, corev1.ConditionFalse)
s.NoError(err, "Failed to find degraded condition")
foundMondooAuditConfig, err = s.testCluster.K8sHelper.GetMondooAuditConfigFromCluster(auditConfig.Name, auditConfig.Namespace)
s.NoError(err, "Failed to find MondooAuditConfig")
s.NotContains(foundMondooAuditConfig.Status.Conditions[5].Message, "OOM", "Found OOMKilled message in condition")
s.Len(foundMondooAuditConfig.Status.Conditions[5].AffectedPods, 0, "Found a pod in condition")

// Give the integration a chance to update
time.Sleep(2 * time.Second)

status, err = s.integration.GetStatus(s.ctx)
s.NoError(err, "Failed to get status")
s.Equal("ACTIVE", status)
}

func (s *AuditConfigBaseSuite) testMondooAuditConfigContainers(auditConfig mondoov2.MondooAuditConfig) {
Expand Down
72 changes: 37 additions & 35 deletions tests/integration/audit_config_namespace_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
package integration

import (
"testing"

"github.com/stretchr/testify/suite"
"go.mondoo.com/mondoo-operator/tests/framework/utils"
"go.uber.org/zap"

"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -74,44 +78,44 @@ func (s *AuditConfigCustomNamespaceSuite) TearDownSuite() {
s.AuditConfigBaseSuite.TearDownSuite()
}

/*
func (s *AuditConfigCustomNamespaceSuite) TestReconcile_KubernetesResources() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, true, false, false, false)
auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name
s.testMondooAuditConfigKubernetesResources(auditConfig)
}
func (s *AuditConfigCustomNamespaceSuite) TestReconcile_KubernetesResources() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, true, false, false, false)
auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name
s.testMondooAuditConfigKubernetesResources(auditConfig)
}

func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Containers() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, true, false, false)
auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name
func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Containers() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, true, false, false)
auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name

// Ignore the operator namespace and the scanner namespace because we cannot scan a local image
// Ignore kube-system to speed up the containers test
auditConfig.Spec.Filtering.Namespaces.Exclude = []string{s.ns.Name, s.testCluster.Settings.Namespace, "kube-system"}
s.testMondooAuditConfigContainers(auditConfig)
}
// Ignore the operator namespace and the scanner namespace because we cannot scan a local image
// Ignore kube-system to speed up the containers test
auditConfig.Spec.Filtering.Namespaces.Exclude = []string{s.ns.Name, s.testCluster.Settings.Namespace, "kube-system"}
s.testMondooAuditConfigContainers(auditConfig)
}

func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Nodes() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, true, false)
auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name
s.testMondooAuditConfigNodes(auditConfig)
}
func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Nodes() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, true, false)
auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name
s.testMondooAuditConfigNodes(auditConfig)
}

func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Admission() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, false, true)
auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name
auditConfig.Spec.Admission.ServiceAccountName = s.webhookServiceAccount.Name
s.testMondooAuditConfigAdmission(auditConfig)
}
func (s *AuditConfigCustomNamespaceSuite) TestReconcile_Admission() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, false, true)
auditConfig.Spec.Scanner.ServiceAccountName = s.sa.Name
auditConfig.Spec.Admission.ServiceAccountName = s.webhookServiceAccount.Name
s.testMondooAuditConfigAdmission(auditConfig)
}

func (s *AuditConfigCustomNamespaceSuite) TestReconcile_AdmissionMissingSA() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, false, true)
auditConfig.Spec.Scanner.ServiceAccountName = "missing-serviceaccount"
auditConfig.Spec.Admission.ServiceAccountName = s.webhookServiceAccount.Name
s.testMondooAuditConfigAdmissionMissingSA(auditConfig)
}
func TestAuditConfigCustomNamespaceSuite(t *testing.T) {
s := new(AuditConfigCustomNamespaceSuite)
func (s *AuditConfigCustomNamespaceSuite) TestReconcile_AdmissionMissingSA() {
auditConfig := utils.DefaultAuditConfigMinimal(s.ns.Name, false, false, false, true)
auditConfig.Spec.Scanner.ServiceAccountName = "missing-serviceaccount"
auditConfig.Spec.Admission.ServiceAccountName = s.webhookServiceAccount.Name
s.testMondooAuditConfigAdmissionMissingSA(auditConfig)
}

func TestAuditConfigCustomNamespaceSuite(t *testing.T) {
s := new(AuditConfigCustomNamespaceSuite)
defer func(s *AuditConfigCustomNamespaceSuite) {
HandlePanics(recover(), func() {
if err := s.testCluster.UninstallOperator(); err != nil {
Expand All @@ -126,5 +130,3 @@ func (s *AuditConfigCustomNamespaceSuite) TearDownSuite() {
}(s)
suite.Run(t, s)
}
*/
75 changes: 38 additions & 37 deletions tests/integration/audit_config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ import (
"testing"

"github.com/stretchr/testify/suite"
"go.mondoo.com/mondoo-operator/api/v1alpha2"
"go.mondoo.com/mondoo-operator/tests/framework/utils"
"go.uber.org/zap"
"k8s.io/utils/ptr"
)

type AuditConfigSuite struct {
Expand All @@ -20,50 +22,49 @@ func (s *AuditConfigSuite) TestOOMControllerReporting() {
s.testOOMMondooOperatorController(auditConfig)
}

/*
func (s *AuditConfigSuite) TestReconcile_AllDisabled() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, false)
s.testMondooAuditConfigAllDisabled(auditConfig)
}
func (s *AuditConfigSuite) TestReconcile_AllDisabled() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, false)
s.testMondooAuditConfigAllDisabled(auditConfig)
}

func (s *AuditConfigSuite) TestReconcile_KubernetesResources() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, true, false, false, false)
s.testMondooAuditConfigKubernetesResources(auditConfig)
}
func (s *AuditConfigSuite) TestReconcile_KubernetesResources() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, true, false, false, false)
s.testMondooAuditConfigKubernetesResources(auditConfig)
}

func (s *AuditConfigSuite) TestReconcile_Containers() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, true, false, false)
func (s *AuditConfigSuite) TestReconcile_Containers() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, true, false, false)

// Ignore the operator namespace because we cannot scan a local image
// Ignore kube-system to speed up the containers test
auditConfig.Spec.Filtering.Namespaces.Exclude = []string{s.testCluster.Settings.Namespace, "kube-system"}
s.testMondooAuditConfigContainers(auditConfig)
}
// Ignore the operator namespace because we cannot scan a local image
// Ignore kube-system to speed up the containers test
auditConfig.Spec.Filtering.Namespaces.Exclude = []string{s.testCluster.Settings.Namespace, "kube-system"}
s.testMondooAuditConfigContainers(auditConfig)
}

func (s *AuditConfigSuite) TestReconcile_Nodes() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, true, false)
s.testMondooAuditConfigNodes(auditConfig)
}
func (s *AuditConfigSuite) TestReconcile_Nodes() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, true, false)
s.testMondooAuditConfigNodes(auditConfig)
}

func (s *AuditConfigSuite) TestReconcile_AdmissionPermissive() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true)
s.testMondooAuditConfigAdmission(auditConfig)
}
func (s *AuditConfigSuite) TestReconcile_AdmissionPermissive() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true)
s.testMondooAuditConfigAdmission(auditConfig)
}

func (s *AuditConfigSuite) TestReconcile_AdmissionEnforcing() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true)
auditConfig.Spec.Admission.Mode = v1alpha2.Enforcing
s.testMondooAuditConfigAdmission(auditConfig)
}

func (s *AuditConfigSuite) TestReconcile_AdmissionEnforcing() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true)
auditConfig.Spec.Admission.Mode = v1alpha2.Enforcing
s.testMondooAuditConfigAdmission(auditConfig)
}
func (s *AuditConfigSuite) TestReconcile_AdmissionEnforcingScaleDownScanApi() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true)
auditConfig.Spec.Admission.Mode = v1alpha2.Enforcing
auditConfig.Spec.Admission.Replicas = ptr.To(int32(1))
auditConfig.Spec.Scanner.Replicas = ptr.To(int32(1))
s.testMondooAuditConfigAdmissionScaleDownScanApi(auditConfig)
}

func (s *AuditConfigSuite) TestReconcile_AdmissionEnforcingScaleDownScanApi() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, false, false, false, true)
auditConfig.Spec.Admission.Mode = v1alpha2.Enforcing
auditConfig.Spec.Admission.Replicas = ptr.To(int32(1))
auditConfig.Spec.Scanner.Replicas = ptr.To(int32(1))
s.testMondooAuditConfigAdmissionScaleDownScanApi(auditConfig)
}
*/
func TestAuditConfigSuite(t *testing.T) {
s := new(AuditConfigSuite)
defer func(s *AuditConfigSuite) {
Expand Down
7 changes: 5 additions & 2 deletions tests/integration/audit_config_upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@
package integration

import (
"testing"

"github.com/stretchr/testify/suite"
"go.mondoo.com/mondoo-operator/tests/framework/installer"
"go.mondoo.com/mondoo-operator/tests/framework/utils"
"go.uber.org/zap"
)

type AuditConfigUpgradeSuite struct {
Expand All @@ -22,7 +27,6 @@ func (s *AuditConfigUpgradeSuite) TearDownSuite() {
s.NoError(s.spaceClient.Delete(s.ctx))
}

/*
func (s *AuditConfigUpgradeSuite) TestUpgradePreviousReleaseToLatest() {
auditConfig := utils.DefaultAuditConfigMinimal(s.testCluster.Settings.Namespace, true, false, true, false)
s.testUpgradePreviousReleaseToLatest(auditConfig)
Expand All @@ -45,4 +49,3 @@ func TestAuditConfigUpgradeSuite(t *testing.T) {
}(s)
suite.Run(t, s)
}
*/

0 comments on commit 4a26977

Please sign in to comment.