test

Author: abhishekdwivedi3060

api/v1/aerospikecluster_types.go

@@ -284,7 +284,8 @@ type RackConfig struct { //nolint:govet // for readability
 	// +optional
 	RollingUpdateBatchSize *intstr.IntOrString `json:"rollingUpdateBatchSize,omitempty"`
 
-	MaxUnavailable int `json:"maxUnavailable,omitempty"`
+	// +optional
+	MaxIgnorableFailedPods int `json:"maxIgnorableFailedPods,omitempty"`
 }
 
 // Rack specifies single rack config

config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml

@@ -4603,7 +4603,7 @@ spec:
                   Aerospike cluster. Pods will be deployed in given racks based on
                   given configuration
                 properties:
-                  maxUnavailable:
+                  maxIgnorableFailedPods:
                     type: integer
                   namespaces:
                     description: List of Aerospike namespaces for which rack feature
@@ -13352,7 +13352,7 @@ spec:
                   given configuration
                 nullable: true
                 properties:
-                  maxUnavailable:
+                  maxIgnorableFailedPods:
                     type: integer
                   namespaces:
                     description: List of Aerospike namespaces for which rack feature

controllers/pod.go

@@ -671,25 +671,46 @@ func (r *SingleClusterReconciler) getIgnorablePods(racksToDelete []asdbv1.Rack,
 
 	for idx := range configureRacks {
 		rack := &configureRacks[idx]
-		failedAllowed := r.aeroCluster.Spec.RackConfig.MaxUnavailable
+		failedAllowed := r.aeroCluster.Spec.RackConfig.MaxIgnorableFailedPods
 
 		podList, err := r.getRackPodList(rack.Rack.ID)
 		if err != nil {
 			return nil, err
 		}
 
+		var (
+			failedPod  []string
+			pendingPod []string
+		)
+
 		for podIdx := range podList.Items {
 			pod := &podList.Items[podIdx]
-			if !utils.IsPodRunningAndReady(pod) && failedAllowed > 0 {
-				ignorablePodNames.Insert(pod.Name)
-				failedAllowed--
+
+			if !utils.IsPodRunningAndReady(pod) {
+				if utils.IsPodReasonUnschedulable(pod) {
+					pendingPod = append(pendingPod, pod.Name)
+					continue
+				}
+
+				failedPod = append(failedPod, pod.Name)
 			}
 		}
+
+		// prepend pendingPod to failedPod
+		failedPod = append(pendingPod, failedPod...)
+
+		for podIdx := range failedPod {
+			if failedAllowed <= 0 {
+				break
+			}
+
+			ignorablePodNames.Insert(failedPod[podIdx])
+			failedAllowed--
+		}
 	}
 
 	return ignorablePodNames, nil
 }
-
 func (r *SingleClusterReconciler) getPodsPVCList(
 	podNames []string, rackID int,
 ) ([]corev1.PersistentVolumeClaim, error) {

go.mod

@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/aerospike/aerospike-client-go/v6 v6.14.0
-	github.com/aerospike/aerospike-management-lib v0.0.0-20231107182540-fef71e1f5946
+	github.com/aerospike/aerospike-management-lib v0.0.0-20231129055344-b6aff63f1dbb
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
 	github.com/evanphx/json-patch v4.12.0+incompatible
 	github.com/go-logr/logr v1.2.4

go.sum

@@ -600,6 +600,8 @@ github.com/aerospike/aerospike-client-go/v6 v6.14.0 h1:Z3FcGWJda1sagzdc6Akz4EJ13
 github.com/aerospike/aerospike-client-go/v6 v6.14.0/go.mod h1:/0Wm81GhMqem+9flWcpazPKoRfjFeG6WrQdXGiMNi0A=
 github.com/aerospike/aerospike-management-lib v0.0.0-20231107182540-fef71e1f5946 h1:wwCzPj4qk4EfdISK6tzNoEwSLg9vbeqBloNmhfB8mNo=
 github.com/aerospike/aerospike-management-lib v0.0.0-20231107182540-fef71e1f5946/go.mod h1:LPOsGG8okRSH4hN9Y8VXFzsfIpBDj2WKEsI/f6wxwaw=
+github.com/aerospike/aerospike-management-lib v0.0.0-20231129055344-b6aff63f1dbb h1:ykX3ElBNT/VOUhw/+5+jiFnWw3LSbPfl6eRrhQzBBFk=
+github.com/aerospike/aerospike-management-lib v0.0.0-20231129055344-b6aff63f1dbb/go.mod h1:LPOsGG8okRSH4hN9Y8VXFzsfIpBDj2WKEsI/f6wxwaw=
 github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
 github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=

pkg/utils/pod.go

@@ -32,7 +32,7 @@ func CheckPodFailed(pod *corev1.Pod) error {
 		return fmt.Errorf("pod %s has failed status", pod.Name)
 	}
 
-	if pod.Status.Phase == corev1.PodPending && isPodReasonUnschedulable(pod) {
+	if pod.Status.Phase == corev1.PodPending && IsPodReasonUnschedulable(pod) {
 		return fmt.Errorf("pod %s is in unschedulable state", pod.Name)
 	}
 
@@ -210,7 +210,7 @@ func isPodError(reason string) bool {
 	return strings.HasSuffix(reason, "Error")
 }
 
-func isPodReasonUnschedulable(pod *corev1.Pod) bool {
+func IsPodReasonUnschedulable(pod *corev1.Pod) bool {
 	for _, condition := range pod.Status.Conditions {
 		if condition.Type == corev1.PodScheduled && condition.Reason == corev1.PodReasonUnschedulable {
 			return true

test/cluster_test.go

@@ -170,7 +170,7 @@ func clusterWithIgnorePodList(ctx goctx.Context) {
 					By("Upgrade version")
 					aeroCluster, err = getCluster(k8sClient, ctx, clusterNamespacedName)
 					Expect(err).ToNot(HaveOccurred())
-					newImage := baseImage + ":6.4.0.4"
+					newImage := baseImage + ":7.0.0.0_2"
 					aeroCluster.Spec.Image = newImage
 					err = updateCluster(k8sClient, ctx, aeroCluster)
 					Expect(err).ToNot(HaveOccurred())