[KO-338] Making the aerospike-init image namespace configurable

Jenkinsfile

@@ -25,6 +25,7 @@ pipeline {
         BUNDLE_IMG="${OPERATOR_BUNDLE_IMAGE_CANDIDATE_NAME}"
 
         AEROSPIKE_CUSTOM_INIT_REGISTRY="568976754000.dkr.ecr.ap-south-1.amazonaws.com"
+        AEROSPIKE_CUSTOM_INIT_REGISTRY_NAMESPACE="aerospike"
     }
 
     stages {
@@ -92,7 +93,7 @@ pipeline {
                         dir("${env.GO_REPO}") {
                             sh "rsync -aK ${env.WORKSPACE}/../../aerospike-kubernetes-operator-resources/secrets/ config/samples/secrets"
 							sh "set +x; docker login --username AWS  568976754000.dkr.ecr.ap-south-1.amazonaws.com -p \$(aws ecr get-login-password --region ap-south-1); set -x"
-                            sh "./test/test.sh -b ${OPERATOR_BUNDLE_IMAGE_CANDIDATE_NAME} -c ${OPERATOR_CATALOG_IMAGE_CANDIDATE_NAME} -r ${AEROSPIKE_CUSTOM_INIT_REGISTRY}"
+                            sh "./test/test.sh -b ${OPERATOR_BUNDLE_IMAGE_CANDIDATE_NAME} -c ${OPERATOR_CATALOG_IMAGE_CANDIDATE_NAME} -r ${AEROSPIKE_CUSTOM_INIT_REGISTRY} -n ${AEROSPIKE_CUSTOM_INIT_REGISTRY_NAMESPACE}"
 
                         }
                     }

api/v1/aerospikecluster_types.go

@@ -318,6 +318,8 @@ type AerospikeInitContainerSpec struct { //nolint:govet // for readability
 	// ImageRegistry is the name of image registry for aerospike-init container image
 	// ImageRegistry, e.g. docker.io, redhat.access.com
 	ImageRegistry string `json:"imageRegistry,omitempty"`
+	// ImageRegistryNamespace is the name of namespace in registry for aerospike-init container image
+	ImageRegistryNamespace *string `json:"imageRegistryNamespace,omitempty"`
 	// SecurityContext that will be added to aerospike-init container created by operator.
 	SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
 	// Define resources requests and limits for Aerospike init Container.

api/v1/utils.go

@@ -71,6 +71,7 @@ const (
 	AerospikeServerContainerName                   = "aerospike-server"
 	AerospikeInitContainerName                     = "aerospike-init"
 	AerospikeInitContainerRegistryEnvVar           = "AEROSPIKE_KUBERNETES_INIT_REGISTRY"
+	AerospikeInitContainerRegistryNamespaceEnvVar  = "AEROSPIKE_KUBERNETES_INIT_REGISTRY_NAMESPACE"
 	AerospikeInitContainerDefaultRegistry          = "docker.io"
 	AerospikeInitContainerDefaultRegistryNamespace = "aerospike"
 	AerospikeInitContainerDefaultRepoAndTag        = "aerospike-kubernetes-init:2.2.1"
@@ -121,33 +122,63 @@ func GetWorkDirectory(aerospikeConfigSpec AerospikeConfigSpec) string {
 	return DefaultWorkDirectory
 }
 
-func getInitContainerImage(registry string) string {
+func getInitContainerImage(registry, namespace string) string {
 	return fmt.Sprintf(
 		"%s/%s/%s", strings.TrimSuffix(registry, "/"),
-		strings.TrimSuffix(AerospikeInitContainerDefaultRegistryNamespace, "/"),
+		strings.TrimSuffix(namespace, "/"),
 		AerospikeInitContainerDefaultRepoAndTag,
 	)
 }
 
 func GetAerospikeInitContainerImage(aeroCluster *AerospikeCluster) string {
+	registry := getInitContainerImageRegistry(aeroCluster)
+	namespace := getInitContainerImageRegistryNamespace(aeroCluster)
+
+	return getInitContainerImage(registry, namespace)
+}
+
+func getInitContainerImageRegistryNamespace(aeroCluster *AerospikeCluster) string {
+	// Given in CR
+	var namespace *string
+	if aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec != nil {
+		namespace = aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec.ImageRegistryNamespace
+	}
+
+	if namespace == nil {
+		// Given in EnvVar
+		envRegistryNamespace, found := os.LookupEnv(AerospikeInitContainerRegistryNamespaceEnvVar)
+		if found {
+			namespace = &envRegistryNamespace
+		}
+	}
+
+	if namespace == nil {
+		return AerospikeInitContainerDefaultRegistryNamespace
+	}
+
+	return *namespace
+}
+
+func getInitContainerImageRegistry(aeroCluster *AerospikeCluster) string {
 	// Given in CR
 	registry := ""
 	if aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec != nil {
 		registry = aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec.ImageRegistry
 	}
 
-	if registry != "" {
-		return getInitContainerImage(registry)
+	if registry == "" {
+		// Given in EnvVar
+		envRegistry, found := os.LookupEnv(AerospikeInitContainerRegistryEnvVar)
+		if found {
+			registry = envRegistry
+		}
 	}
 
-	// Given in EnvVar
-	registry, found := os.LookupEnv(AerospikeInitContainerRegistryEnvVar)
-	if found {
-		return getInitContainerImage(registry)
+	if registry == "" {
+		return AerospikeInitContainerDefaultRegistry
 	}
 
-	// Use default
-	return getInitContainerImage(AerospikeInitContainerDefaultRegistry)
+	return registry
 }
 
 func ClusterNamespacedName(aeroCluster *AerospikeCluster) string {

config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml

@@ -623,6 +623,10 @@ spec:
                           aerospike-init container image ImageRegistry, e.g. docker.io,
                           redhat.access.com
                         type: string
+                      imageRegistryNamespace:
+                        description: ImageRegistryNamespace is the name of namespace
+                          in registry for aerospike-init container image
+                        type: string
                       resources:
                         description: Define resources requests and limits for Aerospike
                           init Container. Only Memory and Cpu resources can be given
@@ -9996,6 +10000,10 @@ spec:
                           aerospike-init container image ImageRegistry, e.g. docker.io,
                           redhat.access.com
                         type: string
+                      imageRegistryNamespace:
+                        description: ImageRegistryNamespace is the name of namespace
+                          in registry for aerospike-init container image
+                        type: string
                       resources:
                         description: Define resources requests and limits for Aerospike
                           init Container. Only Memory and Cpu resources can be given

config/manager/manager.yaml

@@ -61,6 +61,9 @@ spec:
         - name: AEROSPIKE_KUBERNETES_INIT_REGISTRY
           # this is the registry used to pull aerospike-init image
           value: docker.io
+        - name: AEROSPIKE_KUBERNETES_INIT_REGISTRY_NAMESPACE
+          # this is the namespace in registry used to pull aerospike-init image
+          value: aerospike
       serviceAccountName: controller-manager
 
       terminationGracePeriodSeconds: 10

controllers/rack.go

@@ -1081,15 +1081,6 @@ func (r *SingleClusterReconciler) rollingRestartRack(found *appsv1.StatefulSet,
 		}
 	}
 
-	if len(failedPods) != 0 && r.isAnyPodInImageFailedState(podList, ignorablePodNames) {
-		return found, reconcileError(
-			fmt.Errorf(
-				"cannot Rolling restart AerospikeCluster. " +
-					"A pod is already in failed state due to image related issues",
-			),
-		)
-	}
-
 	err = r.updateSTS(found, rackState)
 	if err != nil {
 		return found, reconcileError(

helm-charts/aerospike-kubernetes-operator/crds/customresourcedefinition_aerospikeclusters.asdb.aerospike.com.yaml

@@ -623,6 +623,10 @@ spec:
                           aerospike-init container image ImageRegistry, e.g. docker.io,
                           redhat.access.com
                         type: string
+                      imageRegistryNamespace:
+                        description: ImageRegistryNamespace is the name of namespace
+                          in registry for aerospike-init container image
+                        type: string
                       resources:
                         description: Define resources requests and limits for Aerospike
                           init Container. Only Memory and Cpu resources can be given
@@ -9996,6 +10000,10 @@ spec:
                           aerospike-init container image ImageRegistry, e.g. docker.io,
                           redhat.access.com
                         type: string
+                      imageRegistryNamespace:
+                        description: ImageRegistryNamespace is the name of namespace
+                          in registry for aerospike-init container image
+                        type: string
                       resources:
                         description: Define resources requests and limits for Aerospike
                           init Container. Only Memory and Cpu resources can be given

helm-charts/aerospike-kubernetes-operator/templates/aerospike-operator-controller-manager-deployment.yaml

@@ -57,6 +57,8 @@ spec:
           value: {{ .Values.watchNamespaces | quote }}
         - name: AEROSPIKE_KUBERNETES_INIT_REGISTRY
           value: {{ .Values.aerospikeKubernetesInitRegistry }}
+        - name: AEROSPIKE_KUBERNETES_INIT_REGISTRY_NAMESPACE
+          value: {{ .Values.aerospikeKubernetesInitRegistryNamespace }}
         {{- if .Values.extraEnv }}
         {{- range $key, $value := .Values.extraEnv }}
         - name: "{{ $key }}"

helm-charts/aerospike-kubernetes-operator/values.yaml

@@ -33,6 +33,9 @@ watchNamespaces: "default"
 # Registry used to pull aerospike-init image
 aerospikeKubernetesInitRegistry: "docker.io"
 
+# Namespace in registry used to pull aerospike-init image
+aerospikeKubernetesInitRegistryNamespace: "aerospike"
+
 ## Resources - limits / requests
 resources:
   limits:

test/podspec_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"reflect"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -16,8 +17,9 @@ import (
 )
 
 var (
-	customInitRegistryEnvVar  = "CUSTOM_INIT_REGISTRY"
-	imagePullSecretNameEnvVar = "IMAGE_PULL_SECRET_NAME" //nolint:gosec // for testing
+	customInitRegistryEnvVar          = "CUSTOM_INIT_REGISTRY"
+	customInitRegistryNamespaceEnvVar = "CUSTOM_INIT_REGISTRY_NAMESPACE"
+	imagePullSecretNameEnvVar         = "IMAGE_PULL_SECRET_NAME" //nolint:gosec // for testing
 )
 
 var _ = Describe(
@@ -427,9 +429,11 @@ var _ = Describe(
 					}
 				})
 
-				It("Should be able to set/update aerospike-init custom registry", func() {
+				It("Should be able to set/update aerospike-init custom registry and namespace", func() {
 					operatorEnvVarRegistry := "docker.io"
+					operatorEnvVarRegistryNamespace := "aerospike"
 					customRegistry := getEnvVar(customInitRegistryEnvVar)
+					customRegistryNamespace := getEnvVar(customInitRegistryNamespaceEnvVar)
 					imagePullSecret := getEnvVar(imagePullSecretNameEnvVar)
 
 					By("Updating imagePullSecret")
@@ -445,26 +449,58 @@ var _ = Describe(
 					err = updateCluster(k8sClient, ctx, aeroCluster)
 					Expect(err).ToNot(HaveOccurred())
 
-					By("Using registry in CR")
+					By("Using registry and namespace in CR")
 					aeroCluster, err = getCluster(k8sClient, ctx, clusterNamespacedName)
 					Expect(err).ToNot(HaveOccurred())
 
 					aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec.ImageRegistry = customRegistry
+					aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec.ImageRegistryNamespace = &customRegistryNamespace
 					err = updateCluster(k8sClient, ctx, aeroCluster)
 					Expect(err).ToNot(HaveOccurred())
 
-					validateImageRegistry(k8sClient, ctx, aeroCluster, customRegistry)
+					validateImageRegistryNamespace(k8sClient, ctx, aeroCluster, customRegistry, customRegistryNamespace)
 
-					By("Using envVar registry")
+					By("Using envVar registry and namespace")
 					aeroCluster, err = getCluster(k8sClient, ctx, clusterNamespacedName)
 					Expect(err).ToNot(HaveOccurred())
 
 					// Empty imageRegistry, should use operator envVar docker.io
 					aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec.ImageRegistry = ""
+					aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec.ImageRegistryNamespace = nil
 					err = updateCluster(k8sClient, ctx, aeroCluster)
 					Expect(err).ToNot(HaveOccurred())
 
-					validateImageRegistry(k8sClient, ctx, aeroCluster, operatorEnvVarRegistry)
+					validateImageRegistryNamespace(k8sClient, ctx, aeroCluster, operatorEnvVarRegistry,
+						operatorEnvVarRegistryNamespace)
+				})
+
+				It("Should be able to recover cluster after setting correct aerospike-init custom registry/namespace", func() {
+					operatorEnvVarRegistry := "docker.io"
+					operatorEnvVarRegistryNamespace := "aerospike"
+					incorrectCustomRegistryNamespace := "incorrectnamespace"
+
+					By("Using incorrect registry in CR")
+					aeroCluster, err := getCluster(k8sClient, ctx, clusterNamespacedName)
+					Expect(err).ToNot(HaveOccurred())
+
+					aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec.ImageRegistryNamespace = &incorrectCustomRegistryNamespace
+					err = updateClusterWithTO(k8sClient, ctx, aeroCluster, time.Minute*1)
+					Expect(err).Should(HaveOccurred())
+
+					validateImageRegistryNamespace(k8sClient, ctx, aeroCluster, operatorEnvVarRegistry,
+						incorrectCustomRegistryNamespace)
+
+					By("Using correct registry name")
+					aeroCluster, err = getCluster(k8sClient, ctx, clusterNamespacedName)
+					Expect(err).ToNot(HaveOccurred())
+
+					// Nil ImageRegistryNamespace, should use operator envVar aerospike
+					aeroCluster.Spec.PodSpec.AerospikeInitContainerSpec.ImageRegistryNamespace = nil
+					err = updateCluster(k8sClient, ctx, aeroCluster)
+					Expect(err).ToNot(HaveOccurred())
+
+					validateImageRegistryNamespace(k8sClient, ctx, aeroCluster, operatorEnvVarRegistry,
+						operatorEnvVarRegistryNamespace)
 				})
 			})
 		Context(
@@ -521,7 +557,6 @@ var _ = Describe(
 				)
 			},
 		)
-
 	},
 )
 
@@ -532,15 +567,17 @@ func getEnvVar(envVar string) string {
 	return envVarVal
 }
 
-func validateImageRegistry(
-	k8sClient client.Client, _ goctx.Context, aeroCluster *asdbv1.AerospikeCluster, registry string,
-) {
+func validateImageRegistryNamespace(k8sClient client.Client, _ goctx.Context, aeroCluster *asdbv1.AerospikeCluster,
+	registry, namespace string) {
 	stsList, err := getSTSList(aeroCluster, k8sClient)
 	Expect(err).ToNot(HaveOccurred())
 
+	expectedImagePrefix := fmt.Sprintf("%s/%s", registry, namespace)
+
 	for stsIndex := range stsList.Items {
 		image := stsList.Items[stsIndex].Spec.Template.Spec.InitContainers[0].Image
-		hasPrefix := strings.HasPrefix(image, registry)
-		Expect(hasPrefix).To(BeTrue(), fmt.Sprintf("expected registry %s, found image %s", registry, image))
+		hasPrefix := strings.HasPrefix(image, expectedImagePrefix)
+		Expect(hasPrefix).To(BeTrue(), fmt.Sprintf("expected registry/namespace %s, found image %s",
+			expectedImagePrefix, image))
 	}
 }

test/test.sh

@@ -11,7 +11,7 @@ set -e
 #  test.sh -c aerospike/aerospike-kubernetes-operator-bundle:1.1.0 -f ".*RackManagement.*" -a "--connect-through-network-type=hostInternal"
 #  test.sh -c <IMAGE> -f "<GINKGO-FOCUS-REGEXP>" -a "<PASS-THROUGHS>"
 
-while getopts "b:c:f:a:r:p:" opt
+while getopts "b:c:f:a:r:p:n:" opt
 do
    case "$opt" in
       b ) BUNDLE="$OPTARG" ;;
@@ -20,13 +20,15 @@ do
       a ) ARGS="$OPTARG" ;;
       r ) REGISTRY="$OPTARG" ;;
       p ) CRED_PATH="$OPTARG" ;;
+      n ) REGISTRY_NAMESPACE="$OPTARG" ;;
 
    esac
 done
 
 # Defaults
 CRED_PATH=${CRED_PATH:-$HOME/.docker/config.json}
 REGISTRY=${REGISTRY:-568976754000.dkr.ecr.ap-south-1.amazonaws.com}
+REGISTRY_NAMESPACE=${REGISTRY_NAMESPACE:-aerospike}
 
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -60,6 +62,7 @@ echo "| Starting tests.... |"
 echo "---------------------"
 
 export CUSTOM_INIT_REGISTRY="$REGISTRY"
+export CUSTOM_INIT_REGISTRY_NAMESPACE="$REGISTRY_NAMESPACE"
 export IMAGE_PULL_SECRET_NAME="$IMAGE_PULL_SECRET"
 
 make test FOCUS="$FOCUS" ARGS="$ARGS"