diff --git a/Makefile b/Makefile index e8a97e32518..306231f887b 100644 --- a/Makefile +++ b/Makefile @@ -229,10 +229,6 @@ test: manifests generate test-go-generate add-k8s-host test-fast ## Run tests. i race: KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GO) test -race $(TEST_PACKAGES) -.PHONY: test-integration -test-integration: manifests generate envtest add-k8s-host ## Run tests. if existing k8s cluster is k3d or minikube, specify EXISTING_CLUSTER_TYPE. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GO) test ./test/integration - .PHONY: test-delve test-delve: manifests generate envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" dlv --listen=:$(DEBUG_PORT) --headless=true --api-version=2 --accept-multiclient test $(TEST_PACKAGES) diff --git a/test/integration/backup_mysql_test.go b/test/integration/backup_mysql_test.go deleted file mode 100644 index dda1e16c812..00000000000 --- a/test/integration/backup_mysql_test.go +++ /dev/null @@ -1,167 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package appstest - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" - "github.com/apecloud/kubeblocks/pkg/constant" - intctrlutil "github.com/apecloud/kubeblocks/pkg/generics" - testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps" - testdp "github.com/apecloud/kubeblocks/pkg/testutil/dataprotection" -) - -var _ = Describe("MySQL data protection function", func() { - const clusterDefName = "test-clusterdef" - const clusterVersionName = "test-clusterversion" - const clusterNamePrefix = "test-cluster" - const scriptConfigName = "test-cluster-mysql-scripts" - const mysqlCompDefName = "replicasets" - const mysqlCompName = "mysql" - const backupPolicyTemplateName = "test-backup-policy-template" - const backupPolicyName = "test-backup-policy" - const backupRemotePVCName = "backup-remote-pvc" - const backupName = "test-backup-job" - - // Cleanups - - cleanAll := func() { - // must wait until resources deleted and no longer exist before the testcases start, - // otherwise if later it needs to create some new resource objects with the same name, - // in race conditions, it will find the existence of old objects, resulting failure to - // create the new objects. - By("clean resources") - - // delete cluster(and all dependent sub-resources), clusterversion and clusterdef - testapps.ClearClusterResources(&testCtx) - - inNS := client.InNamespace(testCtx.DefaultNamespace) - ml := client.HasLabels{testCtx.TestObjLabelKey} - // namespaced - testapps.ClearResources(&testCtx, intctrlutil.OpsRequestSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.ConfigMapSignature, inNS, ml) - testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.BackupSignature, true, inNS) - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.ActionSetSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.RestoreSignature, inNS, ml) - - } - - BeforeEach(cleanAll) - - AfterEach(cleanAll) - - // Testcases - - var ( - clusterDefObj *appsv1alpha1.ClusterDefinition - clusterVersionObj *appsv1alpha1.ClusterVersion - clusterObj *appsv1alpha1.Cluster - clusterKey types.NamespacedName - backupKey types.NamespacedName - ) - - createClusterObj := func() { - By("Create configmap") - _ = testapps.CreateCustomizedObj(&testCtx, "resources/mysql-scripts.yaml", &corev1.ConfigMap{}, - testapps.WithName(scriptConfigName), testCtx.UseDefaultNamespace()) - - By("Create a clusterDef obj") - mode := int32(0755) - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). - AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponentVersion(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - Create(&testCtx).GetObject() - - By("Create a cluster obj") - pvcSpec := testapps.NewPVCSpec("1Gi") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompDefName). - SetReplicas(1). - AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) - - By("check cluster running") - Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { - g.Expect(cluster.Status.Phase).To(Equal(appsv1alpha1.RunningClusterPhase)) - })).Should(Succeed()) - } - - createBackupObj := func() { - By("By creating a actionSet") - actionSet := testapps.CreateCustomizedObj(&testCtx, "backup/actionset.yaml", - &dpv1alpha1.ActionSet{}, testapps.RandomizedObjName()) - - By("By creating a backupPolicy from backupPolicyTemplate: " + backupPolicyTemplateName) - backupPolicyObj := testdp.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). - WithRandomName(). - SetTarget(constant.AppInstanceLabelKey, clusterKey.Name). - SetTargetConnectionCredential(constant.GenerateDefaultConnCredential(clusterKey.Name)). - AddBackupMethod(testdp.BackupMethodName, false, actionSet.Name). - SetBackupMethodVolumeMounts(testapps.DataVolumeName, "/data"). - Create(&testCtx).GetObject() - backupPolicyKey := client.ObjectKeyFromObject(backupPolicyObj) - - By("By create remove pvc") - testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, backupRemotePVCName, clusterKey.Name, - "none", "remote-volume"). - SetAnnotations(map[string]string{}). - SetStorage("1Gi"). - Create(&testCtx) - - By("By check backupPolicy available") - Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, backupPolicy *dpv1alpha1.BackupPolicy) { - g.Expect(backupPolicy.Status.Phase).To(Equal(dpv1alpha1.BackupPolicyAvailable)) - })).Should(Succeed()) - - By("By creating a backup from backupPolicy: " + backupPolicyKey.Name) - backup := testdp.NewBackupFactory(testCtx.DefaultNamespace, backupName). - WithRandomName(). - SetBackupPolicyName(backupPolicyKey.Name). - SetBackupMethod(testdp.BackupMethodName). - Create(&testCtx).GetObject() - backupKey = client.ObjectKeyFromObject(backup) - } - - Context("with MySQL full backup", func() { - BeforeEach(func() { - createClusterObj() - createBackupObj() - }) - - It("should be completed", func() { - Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, backup *dpv1alpha1.Backup) { - g.Expect(backup.Status.Phase).To(Equal(dpv1alpha1.BackupPhaseCompleted)) - })).Should(Succeed()) - }) - }) -}) diff --git a/test/integration/controller_suite_test.go b/test/integration/controller_suite_test.go deleted file mode 100644 index d111e34085a..00000000000 --- a/test/integration/controller_suite_test.go +++ /dev/null @@ -1,376 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package appstest - -import ( - "context" - "go/build" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/go-logr/logr" - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - "go.uber.org/zap/zapcore" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics/server" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - appsv1beta1 "github.com/apecloud/kubeblocks/apis/apps/v1beta1" - dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps" - dpctrl "github.com/apecloud/kubeblocks/controllers/dataprotection" - "github.com/apecloud/kubeblocks/controllers/k8score" - cfgcore "github.com/apecloud/kubeblocks/pkg/configuration/core" - "github.com/apecloud/kubeblocks/pkg/constant" - intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" - "github.com/apecloud/kubeblocks/pkg/testutil" - testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps" - viper "github.com/apecloud/kubeblocks/pkg/viperx" - "github.com/apecloud/kubeblocks/test/testutils" -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var cfg *rest.Config -var k8sClient client.Client -var dynamicClient dynamic.Interface -var testEnv *envtest.Environment -var ctx context.Context -var cancel context.CancelFunc -var testCtx testutil.TestContext -var clusterRecorder record.EventRecorder -var logger logr.Logger - -func init() { - viper.AutomaticEnv() - // viper.Set("ENABLE_DEBUG_LOG", "true") -} - -func TestIntegrationController(t *testing.T) { - if testing.Short() { - t.Skip() - } - - RegisterFailHandler(Fail) - - RunSpecs(t, "Integration Test Suite") -} - -// GetConsensusRoleCountMap gets a role:count map from a consensusSet cluster -func GetConsensusRoleCountMap(testCtx testutil.TestContext, k8sClient client.Client, cluster *appsv1alpha1.Cluster) (roleCountMap map[string]int) { - clusterkey := client.ObjectKeyFromObject(cluster) - stsList := &appsv1.StatefulSetList{} - err := testCtx.Cli.List(testCtx.Ctx, stsList, client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterkey.Name, - }, client.InNamespace(clusterkey.Namespace)) - - roleCountMap = make(map[string]int) - roleCountMap["leader"] = 0 - roleCountMap["follower"] = 0 - roleCountMap["learner"] = 0 - - if err != nil || len(stsList.Items) == 0 { - return roleCountMap - } - - sts := stsList.Items[0] - pods, err := intctrlutil.GetPodListByStatefulSet(testCtx.Ctx, k8sClient, &sts) - - if err != nil { - return roleCountMap - } - - for _, pod := range pods { - role := pod.Labels[constant.RoleLabelKey] - roleCountMap[role]++ - } - - return roleCountMap -} - -func CreateSimpleConsensusMySQLClusterWithConfig( - testCtx testutil.TestContext, - clusterDefName, - clusterVersionName, - clusterName, - mysqlConfigTemplatePath, - mysqlConfigConstraintPath, - mysqlScriptsPath string) ( - *appsv1alpha1.ClusterDefinition, *appsv1alpha1.ClusterVersion, *appsv1alpha1.Cluster) { - const mysqlCompName = "mysql" - const mysqlCompDefName = "mysql" - const mysqlConfigName = "mysql-component-config" - const mysqlConfigConstraintName = "mysql8.0-config-constraints" - const mysqlScriptsConfigName = "apecloud-mysql-scripts" - const mysqlDataVolumeName = "data" - const mysqlConfigVolumeName = "mysql-config" - const mysqlScriptsVolumeName = "scripts" - const mysqlErrorFilePath = "/data/mysql/log/mysqld-error.log" - const mysqlGeneralFilePath = "/data/mysql/log/mysqld.log" - const mysqlSlowlogFilePath = "/data/mysql/log/mysqld-slowquery.log" - - mysqlConsensusType := string(testapps.ConsensusMySQLComponent) - - configmap := testapps.CreateCustomizedObj(&testCtx, - mysqlConfigTemplatePath, &corev1.ConfigMap{}, - testCtx.UseDefaultNamespace(), - testapps.WithLabels( - constant.AppNameLabelKey, clusterName, - constant.AppInstanceLabelKey, clusterName, - constant.KBAppComponentLabelKey, mysqlConsensusType, - constant.CMConfigurationTemplateNameLabelKey, mysqlConfigName, - constant.CMConfigurationConstraintsNameLabelKey, mysqlConfigConstraintName, - constant.CMConfigurationSpecProviderLabelKey, mysqlConfigName, - constant.CMConfigurationTypeLabelKey, constant.ConfigInstanceType, - )) - - _ = testapps.CreateCustomizedObj(&testCtx, mysqlScriptsPath, &corev1.ConfigMap{}, - testapps.WithName(mysqlScriptsConfigName), testCtx.UseDefaultNamespace()) - - By("Create a constraint obj") - constraint := testapps.CreateCustomizedObj(&testCtx, - mysqlConfigConstraintPath, - &appsv1beta1.ConfigConstraint{}) - - mysqlVolumeMounts := []corev1.VolumeMount{ - { - Name: mysqlConfigVolumeName, - MountPath: "/opt/mysql", - }, - { - Name: mysqlScriptsVolumeName, - MountPath: "/scripts", - }, - { - Name: mysqlDataVolumeName, - MountPath: "/data/mysql", - }, - } - - By("Create a clusterDefinition obj") - mode := int32(0755) - clusterDefObj := testapps.NewClusterDefFactory(clusterDefName). - SetConnectionCredential(map[string]string{"username": "root", "password": ""}, nil). - AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). - AddConfigTemplate(mysqlConfigName, configmap.Name, constraint.Name, - testCtx.DefaultNamespace, mysqlConfigVolumeName). - AddScriptTemplate(mysqlScriptsConfigName, mysqlScriptsConfigName, - testCtx.DefaultNamespace, mysqlScriptsVolumeName, &mode). - AddContainerVolumeMounts(testapps.DefaultMySQLContainerName, mysqlVolumeMounts). - AddLogConfig("error", mysqlErrorFilePath). - AddLogConfig("general", mysqlGeneralFilePath). - AddLogConfig("slow", mysqlSlowlogFilePath). - AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(mysqlConfigName), configmap.Name, - cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). - AddContainerEnv(testapps.DefaultMySQLContainerName, corev1.EnvVar{Name: "MYSQL_ALLOW_EMPTY_PASSWORD", Value: "yes"}). - AddContainerEnv(testapps.DefaultMySQLContainerName, corev1.EnvVar{Name: "CLUSTER_START_INDEX", Value: "1"}). - AddContainerEnv(testapps.DefaultMySQLContainerName, corev1.EnvVar{Name: "CLUSTER_ID", Value: "1"}). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj") - clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponentVersion(mysqlCompDefName). - AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(mysqlConfigName), configmap.Name, - cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). - Create(&testCtx).GetObject() - - By("Creating a cluster") - pvcSpec := appsv1alpha1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - } - clusterObj := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, - clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(mysqlCompName, mysqlCompDefName). - SetReplicas(3). - SetEnabledLogs("error", "general", "slow"). - AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - Create(&testCtx).GetObject() - - return clusterDefObj, clusterVersionObj, clusterObj -} - -var _ = BeforeSuite(func() { - if viper.GetBool("ENABLE_DEBUG_LOG") { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), func(o *zap.Options) { - o.TimeEncoder = zapcore.ISO8601TimeEncoder - })) - } - - ctx, cancel = context.WithCancel(context.TODO()) - logger = logf.FromContext(ctx).WithValues() - logger.Info("logger start") - - By("bootstrapping test environment") - var flag = true - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases"), - // use dependent external CRDs. - // resolved by ref: https://github.com/operator-framework/operator-sdk/issues/4434#issuecomment-786794418 - filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "kubernetes-csi/external-snapshotter/", - "client/v6@v6.2.0", "config", "crd")}, - ErrorIfCRDPathMissing: true, - UseExistingCluster: &flag, - } - - var err error - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = appsv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - err = appsv1beta1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - err = dpv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - err = snapshotv1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - // +kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - - dynamicClient, err = testutils.NewFactory().DynamicClient() - Expect(err).NotTo(HaveOccurred()) - Expect(dynamicClient).NotTo(BeNil()) - - // run reconcile - k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, - Metrics: server.Options{BindAddress: "0"}, - Client: client.Options{ - Cache: &client.CacheOptions{ - DisableFor: intctrlutil.GetUncachedObjects(), - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - viper.SetDefault("CERT_DIR", "/tmp/k8s-webhook-server/serving-certs") - viper.SetDefault(constant.KBToolsImage, "apecloud/kubeblocks-tools:latest") - viper.SetDefault("PROBE_SERVICE_PORT", 3501) - viper.SetDefault("PROBE_SERVICE_LOG_LEVEL", "info") - - clusterRecorder = k8sManager.GetEventRecorderFor("db-cluster-controller") - err = (&apps.ClusterReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: clusterRecorder, - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - err = (&apps.ClusterDefinitionReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor("cluster-definition-controller"), - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - err = (&apps.ClusterVersionReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor("cluster-version-controller"), - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - err = (&apps.OpsRequestReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor("ops-request-controller"), - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - err = (&apps.SystemAccountReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor("system-account-controller"), - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - err = (&k8score.EventReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor("event-controller"), - }).SetupWithManager(k8sManager, nil) - Expect(err).ToNot(HaveOccurred()) - - err = (&dpctrl.BackupReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor("backup-controller"), - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - err = (&dpctrl.BackupScheduleReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor("backup-policy-controller"), - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - err = (&dpctrl.ActionSetReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Recorder: k8sManager.GetEventRecorderFor("backup-tool-controller"), - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - // pulling docker images is slow - viper.SetDefault("EventuallyTimeout", time.Second*300) - testCtx = testutil.NewDefaultTestContext(ctx, k8sClient, testEnv) - - go func() { - defer GinkgoRecover() - err = k8sManager.Start(ctx) - Expect(err).ToNot(HaveOccurred(), "failed to run manager") - }() -}) - -var _ = AfterSuite(func() { - cancel() - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) diff --git a/test/integration/mysql_ha_test.go b/test/integration/mysql_ha_test.go deleted file mode 100644 index 8a1f45afc6f..00000000000 --- a/test/integration/mysql_ha_test.go +++ /dev/null @@ -1,210 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package appstest - -import ( - "context" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/controllerutil" - intctrlutil "github.com/apecloud/kubeblocks/pkg/generics" - testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps" - testk8s "github.com/apecloud/kubeblocks/pkg/testutil/k8s" -) - -var _ = Describe("MySQL High-Availability function", func() { - const clusterDefName = "test-clusterdef" - const clusterVersionName = "test-clusterversion" - const clusterNamePrefix = "test-cluster" - const scriptConfigName = "test-cluster-mysql-scripts" - const mysqlCompDefName = "replicasets" - const mysqlCompName = "mysql" - const leader = "leader" - const follower = "follower" - - ctx := context.Background() - - // Cleanups - - cleanEnv := func() { - // must wait until resources deleted and no longer exist before the testcases start, - // otherwise if later it needs to create some new resource objects with the same name, - // in race conditions, it will find the existence of old objects, resulting failure to - // create the new objects. - By("clean resources") - - // delete cluster(and all dependent sub-resources), clusterversion and clusterdef - testapps.ClearClusterResources(&testCtx) - - // delete rest configurations - inNS := client.InNamespace(testCtx.DefaultNamespace) - ml := client.HasLabels{testCtx.TestObjLabelKey} - // namespaced - testapps.ClearResources(&testCtx, intctrlutil.ConfigMapSignature, inNS, ml) - // non-namespaced - testapps.ClearResources(&testCtx, intctrlutil.ConfigConstraintSignature, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, ml) - - } - - BeforeEach(cleanEnv) - - AfterEach(cleanEnv) - - // Testcases - - var ( - clusterDefObj *appsv1alpha1.ClusterDefinition - clusterVersionObj *appsv1alpha1.ClusterVersion - clusterObj *appsv1alpha1.Cluster - clusterKey types.NamespacedName - ) - - getRole := func(svc *corev1.Service) (role string) { - tunnel, err := testk8s.OpenTunnel(svc) - defer func() { - _ = tunnel.Close() - }() - Expect(err).NotTo(HaveOccurred()) - - time.Sleep(time.Second) - - db, err := tunnel.GetMySQLConn() - defer func() { - _ = db.Close() - }() - Expect(err).NotTo(HaveOccurred()) - - if role, err = db.GetRole(ctx); err != nil { - return "" - } - return role - } - - testThreeReplicasAndFailover := func() { - By("Create a cluster obj") - pvcSpec := testapps.NewPVCSpec("1Gi") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompDefName). - SetReplicas(3).AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - - By("Waiting the cluster is created") - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - - By("Checking pods' role label") - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - sts := &stsList.Items[0] - pods, err := controllerutil.GetPodListByStatefulSet(ctx, k8sClient, sts) - Expect(err).To(Succeed()) - // should have 3 pods - Expect(len(pods)).Should(Equal(3)) - // 1 leader - // 2 followers - leaderCount, followerCount := 0, 0 - for _, pod := range pods { - switch pod.Labels[constant.RoleLabelKey] { - case leader: - leaderCount++ - case follower: - followerCount++ - } - } - Expect(leaderCount).Should(Equal(1)) - Expect(followerCount).Should(Equal(2)) - - By("Checking services status") - svcList := &corev1.ServiceList{} - Expect(k8sClient.List(ctx, svcList, client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - }, client.InNamespace(clusterKey.Namespace))).Should(Succeed()) - // we should have both external service and headless service - Expect(len(svcList.Items)).Should(Equal(2)) - var externalSvc corev1.Service - for _, svc := range svcList.Items { - if svc.Spec.ClusterIP != "None" { - externalSvc = svc - } - } - Expect(externalSvc).ShouldNot(BeNil()) - // getRole should be leader through service - Eventually(func() string { - return getRole(&externalSvc) - }).Should(Equal(leader)) - - By("Deleting leader pod") - leaderPod := &corev1.Pod{} - for _, pod := range pods { - if pod.Labels[constant.RoleLabelKey] == leader { - leaderPod = &pod - break - } - } - Expect(k8sClient.Delete(ctx, leaderPod)).Should(Succeed()) - - By("Waiting for pod recovered and new leader elected") - Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(sts), - func(g Gomega, sts *appsv1.StatefulSet) { - g.Expect(sts.Status.AvailableReplicas == 3).To(BeTrue()) - })).Should(Succeed()) - - Eventually(func() string { - return getRole(&externalSvc) - }).Should(Equal(leader)) - } - - // Scenarios - - Context("with MySQL defined as Consensus type and three replicas", func() { - BeforeEach(func() { - By("Create configmap") - _ = testapps.CreateCustomizedObj(&testCtx, "resources/mysql-scripts.yaml", &corev1.ConfigMap{}, - testapps.WithName(scriptConfigName), testCtx.UseDefaultNamespace()) - - By("Create a clusterDef obj") - mode := int32(0755) - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - SetConnectionCredential(map[string]string{"username": "root", "password": ""}, nil). - AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). - AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). - AddContainerEnv(testapps.DefaultMySQLContainerName, corev1.EnvVar{Name: "MYSQL_ALLOW_EMPTY_PASSWORD", Value: "yes"}). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponentVersion(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - Create(&testCtx).GetObject() - - }) - - It("should have one leader pod and two follower pods, and the service routes to the leader pod", func() { - testThreeReplicasAndFailover() - }) - }) -}) diff --git a/test/integration/mysql_reconfigure_test.go b/test/integration/mysql_reconfigure_test.go deleted file mode 100644 index b0ce3404894..00000000000 --- a/test/integration/mysql_reconfigure_test.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package appstest - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/pkg/configuration/core" - "github.com/apecloud/kubeblocks/pkg/controllerutil" - "github.com/apecloud/kubeblocks/pkg/generics" - testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps" - testk8s "github.com/apecloud/kubeblocks/pkg/testutil/k8s" - "github.com/apecloud/kubeblocks/test/testutils" -) - -var _ = Describe("MySQL Reconfigure function", func() { - const clusterDefName = "test-clusterdef" - const clusterVersionName = "test-clusterversion" - const clusterNamePrefix = "test-cluster" - - const mysqlConfigTemplatePath = "resources/mysql-consensus-config-template.yaml" - const mysqlConfigConstraintPath = "resources/mysql-consensus-config-constraint.yaml" - const mysqlScriptsPath = "resources/mysql-consensus-scripts.yaml" - - const leader = "leader" - const follower = "follower" - - // ctx := context.Background() - - // Cleanups - - cleanEnv := func() { - // must wait until resources deleted and no longer exist before the testcases start, - // otherwise if later it needs to create some new resource objects with the same name, - // in race conditions, it will find the existence of old objects, resulting failure to - // create the new objects. - By("clean resources") - - // delete cluster(and all dependent sub-resources), clusterversion and clusterdef - testapps.ClearClusterResources(&testCtx) - - // delete rest configurations - inNS := client.InNamespace(testCtx.DefaultNamespace) - ml := client.HasLabels{testCtx.TestObjLabelKey} - // namespaced - testapps.ClearResources(&testCtx, generics.ConfigMapSignature, inNS, ml) - // non-namespaced - testapps.ClearResources(&testCtx, generics.ConfigConstraintSignature, ml) - testapps.ClearResources(&testCtx, generics.BackupPolicyTemplateSignature, ml) - } - - BeforeEach(cleanEnv) - - AfterEach(cleanEnv) - - // Testcases - - var ( - clusterDefObj *appsv1alpha1.ClusterDefinition - clusterVersionObj *appsv1alpha1.ClusterVersion - clusterObj *appsv1alpha1.Cluster - clusterKey types.NamespacedName - ) - - newReconfigureRequest := func(clusterName string, componentName string, - configName string, configFile string, - parameterKey string, parameterValue *string) (opsRequest *appsv1alpha1.OpsRequest) { - randomOpsName := "reconfigure-ops-" + testCtx.GetRandomStr() - opsRequest = testapps.NewOpsRequestObj(randomOpsName, testCtx.DefaultNamespace, - clusterName, appsv1alpha1.ReconfiguringType) - opsRequest.Spec.Reconfigure = &appsv1alpha1.Reconfigure{ - Configurations: []appsv1alpha1.ConfigurationItem{{ - Name: configName, - Keys: []appsv1alpha1.ParameterConfig{{ - Key: configFile, - Parameters: []appsv1alpha1.ParameterPair{ - { - Key: parameterKey, - Value: parameterValue, - }, - }, - }}, - }}, - ComponentOps: appsv1alpha1.ComponentOps{ComponentName: componentName}, - } - return opsRequest - } - - getClusterConfig := func(clusterObj *appsv1alpha1.Cluster) ( - componentName string, tpl *appsv1alpha1.ComponentConfigSpec, cmObj *corev1.ConfigMap) { - - By("Get configuration information from cluster") - componentName = clusterObj.Spec.ComponentSpecs[0].ComponentDefRef - tpls, err := core.GetConfigTemplatesFromComponent(clusterObj.Spec.ComponentSpecs, - clusterDefObj.Spec.ComponentDefs, clusterVersionObj.Spec.ComponentVersions, componentName) - Expect(err).Should(BeNil()) - Expect(len(tpls) > 0).Should(BeTrue()) - - By("Should have at least one valid config") - validTpls := make([]appsv1alpha1.ComponentConfigSpec, 0, len(tpls)) - for _, tpl := range tpls { - if len(tpl.ConfigConstraintRef) > 0 && len(tpl.TemplateRef) > 0 { - validTpls = append(validTpls, tpl) - } - } - Expect(len(validTpls) > 0).Should(BeTrue()) - - cmObj = &corev1.ConfigMap{} - cmName := core.GetComponentCfgName(clusterObj.Name, componentName, tpls[0].Name) - err = testutils.GetResourceObjectFromGVR(testutils.ConfigmapGVR(), client.ObjectKey{ - Name: cmName, - Namespace: testCtx.DefaultNamespace, - }, dynamicClient, cmObj) - Expect(err).Should(BeNil()) - - return componentName, &validTpls[0], cmObj - } - - testReconfigureThreeReplicas := func() { - By("Create a cluster obj") - clusterName := testapps.GetRandomizedKey("", clusterNamePrefix).Name - clusterDefObj, clusterVersionObj, clusterObj = CreateSimpleConsensusMySQLClusterWithConfig( - testCtx, clusterDefName, clusterVersionName, clusterName, mysqlConfigTemplatePath, mysqlConfigConstraintPath, mysqlScriptsPath) - clusterKey = client.ObjectKeyFromObject(clusterObj) - fmt.Printf("ClusterDefinition:%s ClusterVersion:%s Cluster:%s \n", clusterDefObj.Name, clusterVersionObj.Name, clusterObj.Name) - - By("Waiting the cluster is created") - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - - By("Checking pods' role label") - sts := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey).Items[0] - pods, err := controllerutil.GetPodListByStatefulSet(testCtx.Ctx, k8sClient, &sts) - Expect(err).To(Succeed()) - Expect(len(pods)).Should(Equal(3)) - - // get role->count map - By("Checking the count of leader and followers, learners are ignored") - roleCountMap := GetConsensusRoleCountMap(testCtx, k8sClient, clusterObj) - Expect(roleCountMap[leader]).Should(Equal(1)) - Expect(roleCountMap[follower]).Should(Equal(2)) - - By("Checking the cluster config") - componentName, tpl, cmObj := getClusterConfig(clusterObj) - configFile := "" - // get first config file - for k := range cmObj.Data { - configFile = k - break - } - - By("Issue a restart load reconfigure OpsRequest - max_connections") - pKey := "max_connections" - pValue := "2000" - reconfigureOpsRequest := newReconfigureRequest(clusterObj.Name, componentName, - tpl.Name, configFile, pKey, &pValue) - Expect(testCtx.CreateObj(testCtx.Ctx, reconfigureOpsRequest)).Should(Succeed()) - - By("Checking ReconfigureOpsRequest is running") - opsKey := types.NamespacedName{Name: reconfigureOpsRequest.Name, Namespace: testCtx.DefaultNamespace} - Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) - - By("Checking Cluster and changed component phase is Reconfiguring") - Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { - g.Expect(cluster.Status.Phase).To(Equal(appsv1alpha1.UpdatingClusterPhase)) // appsv1alpha1.ReconfiguringPhase - g.Expect(cluster.Status.Components[componentName].Phase).To(Equal(appsv1alpha1.UpdatingClusterCompPhase)) // appsv1alpha1.ReconfiguringPhase - // TODO: add status condition check - })).Should(Succeed()) - - By("Issue another reconfigure OpsRequest that will fail - innodb_read_io_threads") - pKey = "innodb_read_io_threads" - pValue = "2" - reconfigureOpsRequest = newReconfigureRequest(clusterObj.Name, componentName, - tpl.Name, configFile, pKey, &pValue) - Expect(testCtx.CreateObj(testCtx.Ctx, reconfigureOpsRequest)).ShouldNot(Succeed()) - } - - // Scenarios - Context("with MySQL defined as Consensus type and three replicas", func() { - It("should update config with opsrequest in restart mode or dynamic loading mode", func() { - testReconfigureThreeReplicas() - }) - }) -}) diff --git a/test/integration/mysql_scale_test.go b/test/integration/mysql_scale_test.go deleted file mode 100644 index da64e2cde58..00000000000 --- a/test/integration/mysql_scale_test.go +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package appstest - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/pkg/constant" - intctrlutil "github.com/apecloud/kubeblocks/pkg/generics" - testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps" - testk8s "github.com/apecloud/kubeblocks/pkg/testutil/k8s" -) - -var _ = Describe("MySQL Scaling function", func() { - const clusterDefName = "test-clusterdef" - const clusterVersionName = "test-clusterversion" - const clusterNamePrefix = "test-cluster" - const scriptConfigName = "test-cluster-mysql-scripts" - const mysqlCompDefName = "replicasets" - const mysqlCompName = "mysql" - - // Cleanups - - cleanAll := func() { - // must wait until resources deleted and no longer exist before the testcases start, - // otherwise if later it needs to create some new resource objects with the same name, - // in race conditions, it will find the existence of old objects, resulting failure to - // create the new objects. - By("clean resources") - - // delete cluster(and all dependent sub-resources), clusterversion and clusterdef - testapps.ClearClusterResources(&testCtx) - - inNS := client.InNamespace(testCtx.DefaultNamespace) - ml := client.HasLabels{testCtx.TestObjLabelKey} - // namespaced - testapps.ClearResources(&testCtx, intctrlutil.OpsRequestSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.ConfigMapSignature, inNS, ml) - } - - BeforeEach(cleanAll) - - AfterEach(cleanAll) - - // Testcases - - var ( - clusterDefObj *appsv1alpha1.ClusterDefinition - clusterVersionObj *appsv1alpha1.ClusterVersion - clusterObj *appsv1alpha1.Cluster - clusterKey types.NamespacedName - ) - - testVerticalScaleCPUAndMemory := func() { - const opsName = "mysql-verticalscaling" - - By("Create a cluster obj") - resources := corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("800m"), - "memory": resource.MustParse("512Mi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("500m"), - "memory": resource.MustParse("256Mi"), - }, - } - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompDefName). - SetResources(resources).SetReplicas(1). - Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) - - By("check cluster running") - Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { - g.Expect(cluster.Status.Phase).To(Equal(appsv1alpha1.RunningClusterPhase)) - })).Should(Succeed()) - - By("send VerticalScalingOpsRequest successfully") - opsKey := types.NamespacedName{Name: opsName, Namespace: testCtx.DefaultNamespace} - verticalScalingOpsRequest := testapps.NewOpsRequestObj(opsKey.Name, opsKey.Namespace, - clusterObj.Name, appsv1alpha1.VerticalScalingType) - verticalScalingOpsRequest.Spec.TTLSecondsAfterSucceed = 0 - verticalScalingOpsRequest.Spec.VerticalScalingList = []appsv1alpha1.VerticalScaling{ - { - ComponentOps: appsv1alpha1.ComponentOps{ComponentName: mysqlCompName}, - ResourceRequirements: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("400m"), - "memory": resource.MustParse("300Mi"), - }, - }, - }, - } - Expect(testCtx.CreateObj(testCtx.Ctx, verticalScalingOpsRequest)).Should(Succeed()) - - By("check VerticalScalingOpsRequest succeed") - Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(verticalScalingOpsRequest), - func(g Gomega, ops *appsv1alpha1.OpsRequest) { - g.Expect(ops.Status.Phase == appsv1alpha1.OpsSucceedPhase).To(BeTrue()) - })).Should(Succeed()) - - By("check cluster resource requirements changed") - Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, fetched *appsv1alpha1.Cluster) { - g.Expect(fetched.Spec.ComponentSpecs[0].Resources.Requests).To(Equal( - verticalScalingOpsRequest.Spec.VerticalScalingList[0].Requests)) - })).Should(Succeed()) - - By("check OpsRequest reclaimed after ttl") - Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func(lopsReq *appsv1alpha1.OpsRequest) { - lopsReq.Spec.TTLSecondsAfterSucceed = 1 - })).Should(Succeed()) - - By("OpsRequest reclaimed after ttl") - Eventually(func() error { - return k8sClient.Get(testCtx.Ctx, client.ObjectKeyFromObject(verticalScalingOpsRequest), verticalScalingOpsRequest) - }).Should(Satisfy(apierrors.IsNotFound)) - } - - testVerticalScaleStorage := func() { - oldStorageValue := resource.MustParse("1Gi") - newStorageValue := resource.MustParse("2Gi") - - By("Check StorageClass") - defaultStorageClass := testk8s.GetDefaultStorageClass(&testCtx) - if defaultStorageClass == nil { - Skip("No default StorageClass found") - } else if !(defaultStorageClass.AllowVolumeExpansion != nil && *defaultStorageClass.AllowVolumeExpansion) { - Skip("Default StorageClass doesn't allow resize") - } - - By("Create a cluster obj with both log and data volume of 1GB size") - dataPvcSpec := testapps.NewPVCSpec(oldStorageValue.String()) - logPvcSpec := dataPvcSpec - logPvcSpec.StorageClassName = &defaultStorageClass.Name - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompDefName). - AddVolumeClaimTemplate(testapps.DataVolumeName, dataPvcSpec). - AddVolumeClaimTemplate(testapps.LogVolumeName, logPvcSpec). - Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) - - By("Check the replicas") - Eventually(func(g Gomega) { - stsList := &appsv1.StatefulSetList{} - g.Expect(k8sClient.List(testCtx.Ctx, stsList, client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - }, client.InNamespace(clusterKey.Namespace))).To(Succeed()) - g.Expect(len(stsList.Items) > 0).To(BeTrue()) - sts := &stsList.Items[0] - g.Expect(sts.Spec.Replicas).NotTo(BeNil()) - g.Expect(sts.Status.AvailableReplicas).To(Equal(*sts.Spec.Replicas)) - }).Should(Succeed()) - - By("Check the pvc") - Eventually(func() bool { - pvcList := &corev1.PersistentVolumeClaimList{} - Expect(k8sClient.List(testCtx.Ctx, pvcList, client.InNamespace(clusterKey.Namespace))).Should(Succeed()) - return len(pvcList.Items) != 0 - }).Should(BeTrue()) - - By("Update volume size") - Eventually(testapps.GetAndChangeObj(&testCtx, clusterKey, func(fetched *appsv1alpha1.Cluster) { - comp := &fetched.Spec.ComponentSpecs[0] - comp.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = newStorageValue - comp.VolumeClaimTemplates[1].Spec.Resources.Requests[corev1.ResourceStorage] = newStorageValue - })).Should(Succeed()) - - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) - - By("Checking the PVC") - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - for _, sts := range stsList.Items { - for _, vct := range sts.Spec.VolumeClaimTemplates { - for i := *sts.Spec.Replicas - 1; i >= 0; i-- { - pvc := &corev1.PersistentVolumeClaim{} - pvcKey := types.NamespacedName{ - Namespace: clusterKey.Namespace, - Name: fmt.Sprintf("%s-%s-%d", vct.Name, sts.Name, i), - } - Expect(k8sClient.Get(testCtx.Ctx, pvcKey, pvc)).Should(Succeed()) - Expect(pvc.Spec.Resources.Requests[corev1.ResourceStorage]).To(Equal(newStorageValue)) - } - } - } - } - - // Scenarios - - Context("with MySQL defined as a stateful component", func() { - BeforeEach(func() { - _ = testapps.CreateCustomizedObj(&testCtx, "resources/mysql-scripts.yaml", &corev1.ConfigMap{}, - testapps.WithName(scriptConfigName), testCtx.UseDefaultNamespace()) - - By("Create a clusterDef obj") - mode := int32(0755) - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). - AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponentVersion(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - Create(&testCtx).GetObject() - }) - - It("should handle VerticalScalingOpsRequest and change Cluster's cpu&memory requirements", func() { - testVerticalScaleCPUAndMemory() - }) - - It("should handle PVC resize requests if cluster has storage class which enables dynamic-provisioning", func() { - testVerticalScaleStorage() - }) - }) - - Context("with MySQL defined as a consensus component", func() { - BeforeEach(func() { - By("Create configmap") - _ = testapps.CreateCustomizedObj(&testCtx, "resources/mysql-scripts.yaml", &corev1.ConfigMap{}, - testapps.WithName(scriptConfigName), testCtx.UseDefaultNamespace()) - - By("Create a clusterDef obj") - mode := int32(0755) - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). - AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponentVersion(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - Create(&testCtx).GetObject() - }) - - It("should handle VerticalScalingOpsRequest and change Cluster's cpu&memory requirements", func() { - testVerticalScaleCPUAndMemory() - }) - - It("should handle PVC resize requests if cluster has storage class which enables dynamic-provisioning", func() { - testVerticalScaleStorage() - }) - }) -}) diff --git a/test/integration/redis_hscale_test.go b/test/integration/redis_hscale_test.go deleted file mode 100644 index 16752cc3328..00000000000 --- a/test/integration/redis_hscale_test.go +++ /dev/null @@ -1,193 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package appstest - -import ( - "fmt" - "strconv" - "strings" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/controllerutil" - intctrlutil "github.com/apecloud/kubeblocks/pkg/generics" - testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps" - testk8s "github.com/apecloud/kubeblocks/pkg/testutil/k8s" -) - -var _ = Describe("Redis Horizontal Scale function", func() { - - const clusterDefName = "test-clusterdef" - const clusterVersionName = "test-clusterversion" - const clusterNamePrefix = "test-cluster" - - const scriptConfigName = "test-cluster-redis-scripts" - const primaryConfigName = "redis-primary-config" - const secondaryConfigName = "redis-secondary-config" - - const replicas = 3 - - // Cleanups - - cleanEnv := func() { - // must wait until resources deleted and no longer exist before the testcases start, - // otherwise if later it needs to create some new resource objects with the same name, - // in race conditions, it will find the existence of old objects, resulting failure to - // create the new objects. - By("clean resources") - - // delete cluster(and all dependent sub-resources), clusterversion and clusterdef - testapps.ClearClusterResources(&testCtx) - - // delete rest configurations - inNS := client.InNamespace(testCtx.DefaultNamespace) - ml := client.HasLabels{testCtx.TestObjLabelKey} - // namespaced - testapps.ClearResources(&testCtx, intctrlutil.ConfigMapSignature, inNS, ml) - // non-namespaced - testapps.ClearResources(&testCtx, intctrlutil.ConfigConstraintSignature, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, ml) - - } - - BeforeEach(cleanEnv) - - AfterEach(cleanEnv) - - // Testcases - - var ( - clusterDefObj *appsv1alpha1.ClusterDefinition - clusterVersionObj *appsv1alpha1.ClusterVersion - clusterObj *appsv1alpha1.Cluster - clusterKey types.NamespacedName - ) - - testReplicationRedisHorizontalScale := func() { - - By("Mock a cluster obj with replication workloadType.") - pvcSpec := testapps.NewPVCSpec("1Gi") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompSpecName, testapps.DefaultRedisCompDefName). - SetReplicas(replicas).AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - - By("Waiting for cluster creation") - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) - - By("Waiting for the cluster to be running") - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - - By("Checking statefulSet number") - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - Expect(len(stsList.Items)).Should(BeEquivalentTo(1)) - - By("Checking pods number and role label in StatefulSet") - podList, err := controllerutil.GetPodListByStatefulSet(ctx, k8sClient, &stsList.Items[0]) - Expect(err).To(Succeed()) - Expect(len(podList)).Should(BeEquivalentTo(replicas)) - for _, pod := range podList { - if strings.HasSuffix(pod.Name, strconv.Itoa(testapps.DefaultReplicationCandidateIndex)) { - Expect(pod.Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(constant.Primary)) - } else { - Expect(pod.Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(constant.Secondary)) - } - } - - By("Checking services status") - svcList := &corev1.ServiceList{} - Expect(k8sClient.List(ctx, svcList, client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - }, client.InNamespace(clusterKey.Namespace))).Should(Succeed()) - // we should have both external service and headless service - Expect(len(svcList.Items)).Should(Equal(2)) - var externalSvc corev1.Service - for _, svc := range svcList.Items { - if svc.Spec.ClusterIP != "None" { - externalSvc = svc - } - } - Expect(externalSvc).ShouldNot(BeNil()) - - for _, newReplicas := range []int32{4, 2, 7, 1} { - By(fmt.Sprintf("horizontal scale out to %d", newReplicas)) - Expect(testapps.ChangeObj(&testCtx, clusterObj, func(lcluster *appsv1alpha1.Cluster) { - lcluster.Spec.ComponentSpecs[0].Replicas = newReplicas - })).Should(Succeed()) - - By("Wait for the cluster to be running") - Consistently(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - } - } - - // Scenarios - - Context("with Redis defined as replication Type and doing Horizontal scale", func() { - BeforeEach(func() { - _ = testapps.CreateCustomizedObj(&testCtx, "resources/redis-scripts.yaml", &corev1.ConfigMap{}, - testapps.WithName(scriptConfigName), testCtx.UseDefaultNamespace()) - - _ = testapps.CreateCustomizedObj(&testCtx, "resources/redis-primary-config-template.yaml", &corev1.ConfigMap{}, - testapps.WithName(primaryConfigName), testCtx.UseDefaultNamespace()) - - _ = testapps.CreateCustomizedObj(&testCtx, "resources/redis-secondary-config-template.yaml", &corev1.ConfigMap{}, - testapps.WithName(secondaryConfigName), testCtx.UseDefaultNamespace()) - - replicationRedisConfigVolumeMounts := []corev1.VolumeMount{ - { - Name: constant.Primary, - MountPath: "/etc/conf/primary", - }, - { - Name: constant.Secondary, - MountPath: "/etc/conf/secondary", - }, - } - - By("Create a clusterDefinition obj with replication workloadType.") - mode := int32(0755) - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). - AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). - AddConfigTemplate(primaryConfigName, primaryConfigName, "", testCtx.DefaultNamespace, constant.Primary). - AddConfigTemplate(secondaryConfigName, secondaryConfigName, "", testCtx.DefaultNamespace, constant.Secondary). - AddInitContainerVolumeMounts(testapps.DefaultRedisInitContainerName, replicationRedisConfigVolumeMounts). - AddContainerVolumeMounts(testapps.DefaultRedisContainerName, replicationRedisConfigVolumeMounts). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj with replication workloadType.") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponentVersion(testapps.DefaultRedisCompDefName). - AddInitContainerShort(testapps.DefaultRedisInitContainerName, testapps.DefaultRedisImageName). - AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). - Create(&testCtx).GetObject() - }) - - It("Should success with one primary and x secondaries when changes the number of replicas", func() { - testReplicationRedisHorizontalScale() - }) - }) -})