From 7b0e981aa71499cde29bffcf3b16738d9f60e534 Mon Sep 17 00:00:00 2001 From: Abhisek Dwivedi Date: Wed, 13 Mar 2024 22:14:36 +0530 Subject: [PATCH] Fixed intermittent testcase failures --- ...rnetes-operator.clusterserviceversion.yaml | 7 ++++-- test/cluster_helper.go | 20 ++++++++++++++-- test/cluster_test.go | 24 ++++++++++++------- test/large_reconcile_test.go | 9 ++++--- test/sample_files_test.go | 2 +- test/utils.go | 18 ++++++++------ 6 files changed, 57 insertions(+), 23 deletions(-) diff --git a/config/manifests/bases/aerospike-kubernetes-operator.clusterserviceversion.yaml b/config/manifests/bases/aerospike-kubernetes-operator.clusterserviceversion.yaml index e9cec80a0..64dcdffae 100644 --- a/config/manifests/bases/aerospike-kubernetes-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/aerospike-kubernetes-operator.clusterserviceversion.yaml @@ -51,12 +51,15 @@ spec: displayName: Server Image path: image - description: K8sNodeBlockList is a list of Kubernetes nodes which are not - used for Aerospike pods. + used for Aerospike pods. Pods are not scheduled on these nodes. Pods are + migrated from these nodes if already present. This is useful for the maintenance + of Kubernetes nodes. displayName: Kubernetes Node BlockList path: k8sNodeBlockList - description: MaxUnavailable is the percentage/number of pods that can be allowed to go down or unavailable before application disruption. This value is used - to create PodDisruptionBudget. Defaults to 1. + to create PodDisruptionBudget. Defaults to 1. Refer Aerospike documentation + for more details. displayName: Max Unavailable path: maxUnavailable - description: Certificates to connect to Aerospike. diff --git a/test/cluster_helper.go b/test/cluster_helper.go index c9199ae79..97a04a351 100644 --- a/test/cluster_helper.go +++ b/test/cluster_helper.go @@ -761,7 +761,7 @@ func deployClusterWithTO( // Wait for aerocluster to reach the desired cluster size. return waitForAerospikeCluster( k8sClient, ctx, aeroCluster, int(aeroCluster.Spec.Size), retryInterval, - timeout, + timeout, []asdbv1.AerospikeClusterPhase{asdbv1.AerospikeClusterCompleted}, ) } @@ -780,6 +780,21 @@ func updateCluster( return updateClusterWithTO(k8sClient, ctx, aeroCluster, getTimeout(aeroCluster.Spec.Size)) } +func updateClusterWithExpectedPhases( + k8sClient client.Client, ctx goctx.Context, + aeroCluster *asdbv1.AerospikeCluster, expectedPhases []asdbv1.AerospikeClusterPhase, +) error { + err := k8sClient.Update(ctx, aeroCluster) + if err != nil { + return err + } + + return waitForAerospikeCluster( + k8sClient, ctx, aeroCluster, int(aeroCluster.Spec.Size), retryInterval, + getTimeout(aeroCluster.Spec.Size), expectedPhases, + ) +} + func updateClusterWithTO( k8sClient client.Client, ctx goctx.Context, aeroCluster *asdbv1.AerospikeCluster, timeout time.Duration, @@ -791,7 +806,7 @@ func updateClusterWithTO( return waitForAerospikeCluster( k8sClient, ctx, aeroCluster, int(aeroCluster.Spec.Size), retryInterval, - timeout, + timeout, []asdbv1.AerospikeClusterPhase{asdbv1.AerospikeClusterCompleted}, ) } @@ -1442,6 +1457,7 @@ func aerospikeClusterCreateUpdateWithTO( return waitForAerospikeCluster( k8sClient, ctx, desired, int(desired.Spec.Size), retryInterval, timeout, + []asdbv1.AerospikeClusterPhase{asdbv1.AerospikeClusterCompleted}, ) } diff --git a/test/cluster_test.go b/test/cluster_test.go index f28ec7709..fc02970bd 100644 --- a/test/cluster_test.go +++ b/test/cluster_test.go @@ -119,7 +119,7 @@ func ScaleDownWithMigrateFillDelay(ctx goctx.Context) { err = waitForAerospikeCluster( k8sClient, ctx, aeroCluster, int(aeroCluster.Spec.Size), retryInterval, - getTimeout(2), + getTimeout(2), []asdbv1.AerospikeClusterPhase{asdbv1.AerospikeClusterCompleted}, ) Expect(err).ToNot(HaveOccurred()) @@ -134,10 +134,12 @@ func ScaleDownWithMigrateFillDelay(ctx goctx.Context) { func clusterWithMaxIgnorablePod(ctx goctx.Context) { var ( - aeroCluster *asdbv1.AerospikeCluster - err error - nodeList = &v1.NodeList{} - podList = &v1.PodList{} + aeroCluster *asdbv1.AerospikeCluster + err error + nodeList = &v1.NodeList{} + podList = &v1.PodList{} + expectedPhases = []asdbv1.AerospikeClusterPhase{ + asdbv1.AerospikeClusterInProgress, asdbv1.AerospikeClusterCompleted} ) clusterNamespacedName := getNamespacedName( @@ -184,7 +186,9 @@ func clusterWithMaxIgnorablePod(ctx goctx.Context) { aeroCluster.Spec.AerospikeConfig.Value["service"].(map[string]interface{})["proto-fd-max"] = int64(18000) - return updateCluster(k8sClient, ctx, aeroCluster) + // As pod is in pending state, CR object will be won't reach the final phase. + // So expectedPhases can be InProgress or Completed + return updateClusterWithExpectedPhases(k8sClient, ctx, aeroCluster, expectedPhases) }, 1*time.Minute).ShouldNot(HaveOccurred()) By("Upgrade version") @@ -192,7 +196,9 @@ func clusterWithMaxIgnorablePod(ctx goctx.Context) { Expect(err).ToNot(HaveOccurred()) newImage := baseImage + ":7.0.0.0_2" aeroCluster.Spec.Image = newImage - err = updateCluster(k8sClient, ctx, aeroCluster) + // As pod is in pending state, CR object will be won't reach the final phase. + // So expectedPhases can be InProgress or Completed + err = updateClusterWithExpectedPhases(k8sClient, ctx, aeroCluster, expectedPhases) Expect(err).ToNot(HaveOccurred()) By("Verify pending pod") @@ -212,7 +218,9 @@ func clusterWithMaxIgnorablePod(ctx goctx.Context) { aeroCluster, err = getCluster(k8sClient, ctx, clusterNamespacedName) Expect(err).ToNot(HaveOccurred()) aeroCluster.Spec.Size-- - err = updateCluster(k8sClient, ctx, aeroCluster) + // As pod is in pending state, CR object will be won't reach the final phase. + // So expectedPhases can be InProgress or Completed + err = updateClusterWithExpectedPhases(k8sClient, ctx, aeroCluster, expectedPhases) Expect(err).ToNot(HaveOccurred()) By("Verify if all pods are running") diff --git a/test/large_reconcile_test.go b/test/large_reconcile_test.go index 6e59732b0..0e8272fe4 100644 --- a/test/large_reconcile_test.go +++ b/test/large_reconcile_test.go @@ -319,7 +319,8 @@ func waitForClusterScaleDown( return false, err } - return isClusterStateValid(aeroCluster, newCluster, replicas), nil + return isClusterStateValid(aeroCluster, newCluster, replicas, + []asdbv1.AerospikeClusterPhase{asdbv1.AerospikeClusterCompleted}), nil }, ) if err != nil { @@ -360,7 +361,8 @@ func waitForClusterRollingRestart( return false, err } - return isClusterStateValid(aeroCluster, newCluster, replicas), nil + return isClusterStateValid(aeroCluster, newCluster, replicas, + []asdbv1.AerospikeClusterPhase{asdbv1.AerospikeClusterCompleted}), nil }, ) if err != nil { @@ -401,7 +403,8 @@ func waitForClusterUpgrade( return false, err } - return isClusterStateValid(aeroCluster, newCluster, replicas), nil + return isClusterStateValid(aeroCluster, newCluster, replicas, + []asdbv1.AerospikeClusterPhase{asdbv1.AerospikeClusterCompleted}), nil }, ) if err != nil { diff --git a/test/sample_files_test.go b/test/sample_files_test.go index 18f638fdd..d7d7827d9 100644 --- a/test/sample_files_test.go +++ b/test/sample_files_test.go @@ -179,7 +179,7 @@ func deployClusterUsingFile(ctx context.Context, filePath string) (*asdbv1.Aeros if err := waitForAerospikeCluster( k8sClient, ctx, aeroCluster, int(aeroCluster.Spec.Size), retryInterval, - getTimeout(aeroCluster.Spec.Size), + getTimeout(aeroCluster.Spec.Size), []asdbv1.AerospikeClusterPhase{asdbv1.AerospikeClusterCompleted}, ); err != nil { return aeroCluster, err } diff --git a/test/utils.go b/test/utils.go index 78207c235..c7ee3982b 100644 --- a/test/utils.go +++ b/test/utils.go @@ -14,6 +14,7 @@ import ( "strings" "time" + set "github.com/deckarep/golang-set/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -236,7 +237,7 @@ func getLabels() map[string]string { func waitForAerospikeCluster( k8sClient client.Client, ctx goctx.Context, aeroCluster *asdbv1.AerospikeCluster, replicas int, - retryInterval, timeout time.Duration, + retryInterval, timeout time.Duration, expectedPhases []asdbv1.AerospikeClusterPhase, ) error { var isValid bool @@ -260,7 +261,7 @@ func waitForAerospikeCluster( return false, err } - isValid = isClusterStateValid(aeroCluster, newCluster, replicas) + isValid = isClusterStateValid(aeroCluster, newCluster, replicas, expectedPhases) return isValid, nil }, ) @@ -277,7 +278,7 @@ func waitForAerospikeCluster( func isClusterStateValid( aeroCluster *asdbv1.AerospikeCluster, - newCluster *asdbv1.AerospikeCluster, replicas int, + newCluster *asdbv1.AerospikeCluster, replicas int, expectedPhases []asdbv1.AerospikeClusterPhase, ) bool { if int(newCluster.Status.Size) != replicas { pkgLog.Info("Cluster size is not correct") @@ -315,8 +316,9 @@ func isClusterStateValid( } pkgLog.Info( - "Cluster pod's image %s not same as spec %s", newCluster.Status.Pods[podName].Image, - aeroCluster.Spec.Image, + fmt.Sprintf("Cluster pod's image %s not same as spec %s", newCluster.Status.Pods[podName].Image, + aeroCluster.Spec.Image, + ), ) } @@ -325,8 +327,10 @@ func isClusterStateValid( return false } - if newCluster.Status.Phase != asdbv1.AerospikeClusterCompleted { - pkgLog.Info("Cluster phase is not set to Completed") + // Validate phase + phaseSet := set.NewSet(expectedPhases...) + if !phaseSet.Contains(newCluster.Status.Phase) { + pkgLog.Info("Cluster phase is not correct") return false }