From ea609543710aba8b38182b7d0545d9e69c98111d Mon Sep 17 00:00:00 2001 From: Tanmay Jain Date: Tue, 6 Aug 2024 12:07:29 +0530 Subject: [PATCH 1/4] Adding support for HPA --- api/v1/aerospikecluster_types.go | 4 +++- .../crd/bases/asdb.aerospike.com_aerospikeclusters.yaml | 8 ++++++++ controllers/reconciler.go | 4 ++++ ...cedefinition_aerospikeclusters.asdb.aerospike.com.yaml | 8 ++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/api/v1/aerospikecluster_types.go b/api/v1/aerospikecluster_types.go index bcee242a2..6a69a1593 100644 --- a/api/v1/aerospikecluster_types.go +++ b/api/v1/aerospikecluster_types.go @@ -722,7 +722,8 @@ type AerospikeClusterStatus struct { //nolint:govet // for readability Pods map[string]AerospikePodStatus `json:"pods" patchStrategy:"strategic"` // Phase denotes the current phase of Aerospike cluster operation. - Phase AerospikeClusterPhase `json:"phase,omitempty"` + Phase AerospikeClusterPhase `json:"phase,omitempty"` + Selector string `json:"selector"` } // AerospikeNetworkType specifies the type of network address to use. @@ -921,6 +922,7 @@ type AerospikePodStatus struct { //nolint:govet // for readability // +kubebuilder:printcolumn:name="HostNetwork",type=boolean,JSONPath=`.spec.podSpec.hostNetwork` // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +//+kubebuilder:subresource:scale:specpath=.spec.size,statuspath=.status.size,selectorpath=.status.selector // AerospikeCluster is the schema for the AerospikeCluster API // +operator-sdk:csv:customresourcedefinitions:displayName="Aerospike Cluster",resources={{Service, v1},{Pod,v1},{StatefulSet,v1}} diff --git a/config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml b/config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml index 1984ff2b2..2f9447451 100644 --- a/config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml +++ b/config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml @@ -18282,6 +18282,8 @@ spec: type: integer type: object type: object + selector: + type: string size: description: Aerospike cluster size format: int32 @@ -18893,11 +18895,17 @@ spec: - skipWorkDirValidate - skipXdrDlogFileValidate type: object + required: + - selector type: object type: object served: true storage: true subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.size + statusReplicasPath: .status.size status: {} - additionalPrinterColumns: - jsonPath: .spec.size diff --git a/controllers/reconciler.go b/controllers/reconciler.go index c39fcd6fa..3f99f3efc 100644 --- a/controllers/reconciler.go +++ b/controllers/reconciler.go @@ -10,6 +10,7 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" k8sRuntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -428,6 +429,9 @@ func (r *SingleClusterReconciler) updateStatus() error { newAeroCluster.Status.IsReadinessProbeEnabled = clusterReadinessEnable } + selector := labels.SelectorFromSet(utils.LabelsForAerospikeCluster(newAeroCluster.Name)) + newAeroCluster.Status.Selector = selector.String() + err = r.patchStatus(newAeroCluster) if err != nil { return fmt.Errorf("error updating status: %w", err) diff --git a/helm-charts/aerospike-kubernetes-operator/crds/customresourcedefinition_aerospikeclusters.asdb.aerospike.com.yaml b/helm-charts/aerospike-kubernetes-operator/crds/customresourcedefinition_aerospikeclusters.asdb.aerospike.com.yaml index 1984ff2b2..2f9447451 100644 --- a/helm-charts/aerospike-kubernetes-operator/crds/customresourcedefinition_aerospikeclusters.asdb.aerospike.com.yaml +++ b/helm-charts/aerospike-kubernetes-operator/crds/customresourcedefinition_aerospikeclusters.asdb.aerospike.com.yaml @@ -18282,6 +18282,8 @@ spec: type: integer type: object type: object + selector: + type: string size: description: Aerospike cluster size format: int32 @@ -18893,11 +18895,17 @@ spec: - skipWorkDirValidate - skipXdrDlogFileValidate type: object + required: + - selector type: object type: object served: true storage: true subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.size + statusReplicasPath: .status.size status: {} - additionalPrinterColumns: - jsonPath: .spec.size From 17ea3a76bf18e9984ce137b376be870cb16ef301 Mon Sep 17 00:00:00 2001 From: Tanmay Jain Date: Fri, 16 Aug 2024 13:10:19 +0530 Subject: [PATCH 2/4] nit --- api/v1/aerospikecluster_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1/aerospikecluster_types.go b/api/v1/aerospikecluster_types.go index 6a69a1593..d9427c54a 100644 --- a/api/v1/aerospikecluster_types.go +++ b/api/v1/aerospikecluster_types.go @@ -922,7 +922,7 @@ type AerospikePodStatus struct { //nolint:govet // for readability // +kubebuilder:printcolumn:name="HostNetwork",type=boolean,JSONPath=`.spec.podSpec.hostNetwork` // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" -//+kubebuilder:subresource:scale:specpath=.spec.size,statuspath=.status.size,selectorpath=.status.selector +// +kubebuilder:subresource:scale:specpath=.spec.size,statuspath=.status.size,selectorpath=.status.selector // AerospikeCluster is the schema for the AerospikeCluster API // +operator-sdk:csv:customresourcedefinitions:displayName="Aerospike Cluster",resources={{Service, v1},{Pod,v1},{StatefulSet,v1}} From 76926b5d51d5c19b6b0244187507d63612188585 Mon Sep 17 00:00:00 2001 From: Tanmay Jain Date: Tue, 3 Dec 2024 16:34:41 +0530 Subject: [PATCH 3/4] Adding testcase for scale subresource. --- test/backup/backup_suite_test.go | 2 +- .../backup_service_suite_test.go | 2 +- test/cluster/autoscaling_test.go | 79 +++++++++++++++++++ test/cluster/suite_test.go | 5 +- test/restore/restore_suite_test.go | 2 +- test/utils.go | 21 +++-- 6 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 test/cluster/autoscaling_test.go diff --git a/test/backup/backup_suite_test.go b/test/backup/backup_suite_test.go index e53be0e8f..c665daab2 100644 --- a/test/backup/backup_suite_test.go +++ b/test/backup/backup_suite_test.go @@ -40,7 +40,7 @@ var _ = BeforeSuite( By("Bootstrapping test environment") var err error - testEnv, _, k8sClient, _, err = test.BootStrapTestEnv(scheme) + testEnv, _, k8sClient, _, _, err = test.BootStrapTestEnv(scheme) Expect(err).NotTo(HaveOccurred()) By("Deploy Backup Service") diff --git a/test/backup_service/backup_service_suite_test.go b/test/backup_service/backup_service_suite_test.go index 3d8ff23c4..1979d2e82 100644 --- a/test/backup_service/backup_service_suite_test.go +++ b/test/backup_service/backup_service_suite_test.go @@ -34,7 +34,7 @@ var _ = BeforeSuite( By("Bootstrapping test environment") var err error - testEnv, _, k8sClient, _, err = test.BootStrapTestEnv(scheme) + testEnv, _, k8sClient, _, _, err = test.BootStrapTestEnv(scheme) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/cluster/autoscaling_test.go b/test/cluster/autoscaling_test.go new file mode 100644 index 000000000..5dcecc593 --- /dev/null +++ b/test/cluster/autoscaling_test.go @@ -0,0 +1,79 @@ +package cluster + +import ( + goctx "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/net/context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = FDescribe("AutoScaler", func() { + ctx := goctx.TODO() + + Context("When doing scale operations", func() { + clusterName := "autoscale" + clusterNamespacedName := getNamespacedName( + clusterName, namespace, + ) + + BeforeEach( + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + }, + ) + + AfterEach( + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + + _ = deleteCluster(k8sClient, ctx, aeroCluster) + }, + ) + + It( + "Should trigger scale up/down via scale subresource", func() { + // Testing over upgrade as it is a long-running operation + By("Scale up the cluster") + + aeroCluster, err := getCluster(k8sClient, ctx, clusterNamespacedName) + Expect(err).ToNot(HaveOccurred()) + + gvr := schema.GroupVersionResource{ + Group: "asdb.aerospike.com", // Replace with your CRD group + Version: "v1", // API version + Resource: "aerospikeclusters", // Replace with your resource + } + + scale, err := dynamicClient.Resource(gvr).Namespace(aeroCluster.Namespace).Get(context.TODO(), + aeroCluster.GetName(), metav1.GetOptions{}, "scale") + Expect(err).ToNot(HaveOccurred()) + + Expect(scale.Object["spec"].(map[string]interface{})["replicas"]).To(Equal(int64(2))) + + scale.Object["spec"].(map[string]interface{})["replicas"] = 3 + + _, err = dynamicClient.Resource(gvr).Namespace(aeroCluster.Namespace).Update(context.TODO(), + scale, metav1.UpdateOptions{}, "scale") + Expect(err).ToNot(HaveOccurred()) + + Eventually( + func() int32 { + aeroCluster, err = getCluster(k8sClient, ctx, clusterNamespacedName) + Expect(err).ToNot(HaveOccurred()) + + return aeroCluster.Spec.Size + }, 1*time.Minute, + ).Should(Equal(int32(3))) + }, + ) + }) +}) diff --git a/test/cluster/suite_test.go b/test/cluster/suite_test.go index d119bc6bb..23091cfbf 100644 --- a/test/cluster/suite_test.go +++ b/test/cluster/suite_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" k8Runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" @@ -51,6 +52,8 @@ var cfg *rest.Config var k8sClientSet *kubernetes.Clientset +var dynamicClient *dynamic.DynamicClient + var projectRoot string var scheme = k8Runtime.NewScheme() @@ -96,7 +99,7 @@ var _ = BeforeSuite( *defaultNetworkType)) var err error - testEnv, cfg, k8sClient, k8sClientSet, err = test.BootStrapTestEnv(scheme) + testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err = test.BootStrapTestEnv(scheme) Expect(err).NotTo(HaveOccurred()) projectRoot, err = getGitRepoRootPath() diff --git a/test/restore/restore_suite_test.go b/test/restore/restore_suite_test.go index e6b52ccad..3b949abc1 100644 --- a/test/restore/restore_suite_test.go +++ b/test/restore/restore_suite_test.go @@ -42,7 +42,7 @@ var _ = BeforeSuite( By("Bootstrapping test environment") var err error - testEnv, _, k8sClient, _, err = test.BootStrapTestEnv(scheme) + testEnv, _, k8sClient, _, _, err = test.BootStrapTestEnv(scheme) Expect(err).NotTo(HaveOccurred()) By("Deploy Backup Service") diff --git a/test/utils.go b/test/utils.go index 8db02085e..b77497c43 100644 --- a/test/utils.go +++ b/test/utils.go @@ -6,6 +6,7 @@ import ( admissionv1 "k8s.io/api/admission/v1" "k8s.io/apimachinery/pkg/runtime" utilRuntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -17,7 +18,7 @@ import ( ) func BootStrapTestEnv(scheme *runtime.Scheme) (testEnv *envtest.Environment, cfg *rest.Config, - k8sClient client.Client, k8sClientSet *kubernetes.Clientset, err error) { + k8sClient client.Client, k8sClientSet *kubernetes.Clientset, dynamicClient *dynamic.DynamicClient, err error) { t := true testEnv = &envtest.Environment{ UseExistingCluster: &t, @@ -26,12 +27,12 @@ func BootStrapTestEnv(scheme *runtime.Scheme) (testEnv *envtest.Environment, cfg cfg, err = testEnv.Start() if err != nil { - return testEnv, cfg, k8sClient, k8sClientSet, err + return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err } if cfg == nil { err = fmt.Errorf("cfg is nil") - return testEnv, cfg, k8sClient, k8sClientSet, err + return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err } utilRuntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -46,20 +47,26 @@ func BootStrapTestEnv(scheme *runtime.Scheme) (testEnv *envtest.Environment, cfg ) if err != nil { - return testEnv, cfg, k8sClient, k8sClientSet, err + return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err } if k8sClient == nil { err = fmt.Errorf("k8sClient is nil") - return testEnv, cfg, k8sClient, k8sClientSet, err + return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err } k8sClientSet = kubernetes.NewForConfigOrDie(cfg) if k8sClientSet == nil { err = fmt.Errorf("k8sClientSet is nil") - return testEnv, cfg, k8sClient, k8sClientSet, err + return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err } - return testEnv, cfg, k8sClient, k8sClientSet, nil + dynamicClient = dynamic.NewForConfigOrDie(cfg) + if dynamicClient == nil { + err = fmt.Errorf("dynamicClient is nil") + return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err + } + + return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, nil } From 920012b10cb34b9f27046a0488bda4d73c3c83eb Mon Sep 17 00:00:00 2001 From: Tanmay Jain Date: Tue, 3 Dec 2024 18:34:31 +0530 Subject: [PATCH 4/4] fix lint --- test/backup/backup_suite_test.go | 2 +- .../backup_service_suite_test.go | 2 +- test/cluster/autoscaling_test.go | 4 ++++ test/cluster/large_reconcile_test.go | 1 + test/cluster/suite_test.go | 5 +---- test/restore/restore_suite_test.go | 2 +- test/utils.go | 21 +++++++------------ 7 files changed, 16 insertions(+), 21 deletions(-) diff --git a/test/backup/backup_suite_test.go b/test/backup/backup_suite_test.go index c665daab2..e53be0e8f 100644 --- a/test/backup/backup_suite_test.go +++ b/test/backup/backup_suite_test.go @@ -40,7 +40,7 @@ var _ = BeforeSuite( By("Bootstrapping test environment") var err error - testEnv, _, k8sClient, _, _, err = test.BootStrapTestEnv(scheme) + testEnv, _, k8sClient, _, err = test.BootStrapTestEnv(scheme) Expect(err).NotTo(HaveOccurred()) By("Deploy Backup Service") diff --git a/test/backup_service/backup_service_suite_test.go b/test/backup_service/backup_service_suite_test.go index 1979d2e82..3d8ff23c4 100644 --- a/test/backup_service/backup_service_suite_test.go +++ b/test/backup_service/backup_service_suite_test.go @@ -34,7 +34,7 @@ var _ = BeforeSuite( By("Bootstrapping test environment") var err error - testEnv, _, k8sClient, _, _, err = test.BootStrapTestEnv(scheme) + testEnv, _, k8sClient, _, err = test.BootStrapTestEnv(scheme) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/cluster/autoscaling_test.go b/test/cluster/autoscaling_test.go index 5dcecc593..7987a6112 100644 --- a/test/cluster/autoscaling_test.go +++ b/test/cluster/autoscaling_test.go @@ -9,6 +9,7 @@ import ( "golang.org/x/net/context" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" ) var _ = FDescribe("AutoScaler", func() { @@ -53,6 +54,9 @@ var _ = FDescribe("AutoScaler", func() { Resource: "aerospikeclusters", // Replace with your resource } + dynamicClient := dynamic.NewForConfigOrDie(cfg) + Expect(dynamicClient).ToNot(BeNil()) + scale, err := dynamicClient.Resource(gvr).Namespace(aeroCluster.Namespace).Get(context.TODO(), aeroCluster.GetName(), metav1.GetOptions{}, "scale") Expect(err).ToNot(HaveOccurred()) diff --git a/test/cluster/large_reconcile_test.go b/test/cluster/large_reconcile_test.go index 9cf625f98..cda6aa6cc 100644 --- a/test/cluster/large_reconcile_test.go +++ b/test/cluster/large_reconcile_test.go @@ -253,6 +253,7 @@ func loadDataInCluster( fmt.Print(strconv.Itoa(i) + ", ") } + fmt.Println("added records") return nil diff --git a/test/cluster/suite_test.go b/test/cluster/suite_test.go index 23091cfbf..d119bc6bb 100644 --- a/test/cluster/suite_test.go +++ b/test/cluster/suite_test.go @@ -26,7 +26,6 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" k8Runtime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" @@ -52,8 +51,6 @@ var cfg *rest.Config var k8sClientSet *kubernetes.Clientset -var dynamicClient *dynamic.DynamicClient - var projectRoot string var scheme = k8Runtime.NewScheme() @@ -99,7 +96,7 @@ var _ = BeforeSuite( *defaultNetworkType)) var err error - testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err = test.BootStrapTestEnv(scheme) + testEnv, cfg, k8sClient, k8sClientSet, err = test.BootStrapTestEnv(scheme) Expect(err).NotTo(HaveOccurred()) projectRoot, err = getGitRepoRootPath() diff --git a/test/restore/restore_suite_test.go b/test/restore/restore_suite_test.go index 3b949abc1..e6b52ccad 100644 --- a/test/restore/restore_suite_test.go +++ b/test/restore/restore_suite_test.go @@ -42,7 +42,7 @@ var _ = BeforeSuite( By("Bootstrapping test environment") var err error - testEnv, _, k8sClient, _, _, err = test.BootStrapTestEnv(scheme) + testEnv, _, k8sClient, _, err = test.BootStrapTestEnv(scheme) Expect(err).NotTo(HaveOccurred()) By("Deploy Backup Service") diff --git a/test/utils.go b/test/utils.go index b77497c43..8db02085e 100644 --- a/test/utils.go +++ b/test/utils.go @@ -6,7 +6,6 @@ import ( admissionv1 "k8s.io/api/admission/v1" "k8s.io/apimachinery/pkg/runtime" utilRuntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -18,7 +17,7 @@ import ( ) func BootStrapTestEnv(scheme *runtime.Scheme) (testEnv *envtest.Environment, cfg *rest.Config, - k8sClient client.Client, k8sClientSet *kubernetes.Clientset, dynamicClient *dynamic.DynamicClient, err error) { + k8sClient client.Client, k8sClientSet *kubernetes.Clientset, err error) { t := true testEnv = &envtest.Environment{ UseExistingCluster: &t, @@ -27,12 +26,12 @@ func BootStrapTestEnv(scheme *runtime.Scheme) (testEnv *envtest.Environment, cfg cfg, err = testEnv.Start() if err != nil { - return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err + return testEnv, cfg, k8sClient, k8sClientSet, err } if cfg == nil { err = fmt.Errorf("cfg is nil") - return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err + return testEnv, cfg, k8sClient, k8sClientSet, err } utilRuntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -47,26 +46,20 @@ func BootStrapTestEnv(scheme *runtime.Scheme) (testEnv *envtest.Environment, cfg ) if err != nil { - return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err + return testEnv, cfg, k8sClient, k8sClientSet, err } if k8sClient == nil { err = fmt.Errorf("k8sClient is nil") - return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err + return testEnv, cfg, k8sClient, k8sClientSet, err } k8sClientSet = kubernetes.NewForConfigOrDie(cfg) if k8sClientSet == nil { err = fmt.Errorf("k8sClientSet is nil") - return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err + return testEnv, cfg, k8sClient, k8sClientSet, err } - dynamicClient = dynamic.NewForConfigOrDie(cfg) - if dynamicClient == nil { - err = fmt.Errorf("dynamicClient is nil") - return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, err - } - - return testEnv, cfg, k8sClient, k8sClientSet, dynamicClient, nil + return testEnv, cfg, k8sClient, k8sClientSet, nil }