From 85e3c3395e7066b2a76ee99b0a7eee0de176fc20 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Fri, 10 Jan 2025 11:01:09 +0800 Subject: [PATCH 1/2] chore: support for switchover in the sharding component by specifying the componentObjectName of the shard --- apis/operations/v1alpha1/opsrequest_types.go | 24 +++-- .../v1alpha1/opsrequest_validation.go | 11 ++- .../v1alpha1/zz_generated.deepcopy.go | 1 - .../operations.kubeblocks.io_opsrequests.yaml | 44 ++++++--- .../operations.kubeblocks.io_opsrequests.yaml | 44 ++++++--- pkg/operations/switchover.go | 93 ++++++++++++++----- pkg/operations/switchover_test.go | 6 +- 7 files changed, 157 insertions(+), 66 deletions(-) diff --git a/apis/operations/v1alpha1/opsrequest_types.go b/apis/operations/v1alpha1/opsrequest_types.go index 7642a853cc0..f70f6d7a418 100644 --- a/apis/operations/v1alpha1/opsrequest_types.go +++ b/apis/operations/v1alpha1/opsrequest_types.go @@ -173,10 +173,6 @@ type SpecificOpsRequest struct { // Lists Switchover objects, each specifying a Component to perform the switchover operation. // // +optional - // +patchMergeKey=componentName - // +patchStrategy=merge,retainKeys - // +listType=map - // +listMapKey=componentName // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="forbidden to update spec.switchover" SwitchoverList []Switchover `json:"switchover,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"componentName"` @@ -239,7 +235,7 @@ type SpecificOpsRequest struct { // ComponentOps specifies the Component to be operated on. type ComponentOps struct { - // Specifies the name of the Component. + // Specifies the name of the Component as defined in the cluster.spec // +kubebuilder:validation:Required ComponentName string `json:"componentName"` } @@ -302,9 +298,16 @@ type Instance struct { TargetNodeName string `json:"targetNodeName,omitempty"` } +// +kubebuilder:validation:XValidation:rule="(has(self.componentName) && !has(self.componentObjectName)) || (!has(self.componentName) && has(self.componentObjectName))",message="need to specified only componentName or componentObjectName" + type Switchover struct { - // Specifies the name of the Component. - ComponentOps `json:",inline"` + // Specifies the name of the Component as defined in the cluster.spec. + // +optional + ComponentName string `json:"componentName,omitempty"` + + // Specifies the name of the Component object. + // +optional + ComponentObjectName string `json:"componentObjectName,omitempty"` // Specifies the instance whose role will be transferred. A typical usage is to transfer the leader role // in a consensus system. @@ -1338,3 +1341,10 @@ func (p *ProgressStatusDetail) SetStatusAndMessage(status ProgressStatus, messag p.Message = message p.Status = status } + +func (s *Switchover) GetComponentName() string { + if len(s.ComponentObjectName) > 0 { + return s.ComponentObjectName + } + return s.ComponentName +} diff --git a/apis/operations/v1alpha1/opsrequest_validation.go b/apis/operations/v1alpha1/opsrequest_validation.go index 57ce7cb3411..63a934890e7 100644 --- a/apis/operations/v1alpha1/opsrequest_validation.go +++ b/apis/operations/v1alpha1/opsrequest_validation.go @@ -499,9 +499,14 @@ func (r *OpsRequest) validateSwitchover(cluster *appsv1.Cluster) error { if len(switchoverList) == 0 { return notEmptyError("spec.switchover") } - compOpsList := make([]ComponentOps, len(switchoverList)) - for i, v := range switchoverList { - compOpsList[i] = v.ComponentOps + compOpsList := make([]ComponentOps, 0) + for _, v := range switchoverList { + if len(v.ComponentName) == 0 { + continue + } + compOpsList = append(compOpsList, ComponentOps{ + ComponentName: v.ComponentName, + }) } if err := r.checkComponentExistence(cluster, compOpsList); err != nil { diff --git a/apis/operations/v1alpha1/zz_generated.deepcopy.go b/apis/operations/v1alpha1/zz_generated.deepcopy.go index a6e0f7bb3c6..54a3466a011 100644 --- a/apis/operations/v1alpha1/zz_generated.deepcopy.go +++ b/apis/operations/v1alpha1/zz_generated.deepcopy.go @@ -1539,7 +1539,6 @@ func (in *SpecificOpsRequest) DeepCopy() *SpecificOpsRequest { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Switchover) DeepCopyInto(out *Switchover) { *out = *in - out.ComponentOps = in.ComponentOps } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Switchover. diff --git a/config/crd/bases/operations.kubeblocks.io_opsrequests.yaml b/config/crd/bases/operations.kubeblocks.io_opsrequests.yaml index 2cdeda3128d..9be03754889 100644 --- a/config/crd/bases/operations.kubeblocks.io_opsrequests.yaml +++ b/config/crd/bases/operations.kubeblocks.io_opsrequests.yaml @@ -176,7 +176,8 @@ spec: items: properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string parameters: description: Specifies the parameters that match the schema @@ -562,7 +563,8 @@ spec: scaling operation. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string scaleIn: description: |- @@ -4046,7 +4048,8 @@ spec: - Logical backups (e.g., 'mysqldump' for MySQL) are unsupported in the current version. type: string componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string inPlace: description: |- @@ -4223,7 +4226,8 @@ spec: configuration. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string configurations: description: |- @@ -4323,7 +4327,8 @@ spec: on. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string required: - componentName @@ -4533,7 +4538,8 @@ spec: on. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string required: - componentName @@ -4554,7 +4560,8 @@ spec: on. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string required: - componentName @@ -4579,7 +4586,11 @@ spec: Refer to ComponentDefinition's Swtichover lifecycle action for more details. type: string componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec. + type: string + componentObjectName: + description: Specifies the name of the Component object. type: string instanceName: description: |- @@ -4587,13 +4598,13 @@ spec: in a consensus system. type: string required: - - componentName - instanceName type: object + x-kubernetes-validations: + - message: need to specified only componentName or componentObjectName + rule: (has(self.componentName) && !has(self.componentObjectName)) + || (!has(self.componentName) && has(self.componentObjectName)) type: array - x-kubernetes-list-map-keys: - - componentName - x-kubernetes-list-type: map x-kubernetes-validations: - message: forbidden to update spec.switchover rule: self == oldSelf @@ -4666,7 +4677,8 @@ spec: maxLength: 64 type: string componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string serviceVersion: description: |- @@ -4727,7 +4739,8 @@ spec: - name x-kubernetes-list-type: map componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string instances: description: Specifies the desired compute resources of the @@ -4839,7 +4852,8 @@ spec: for a volume expansion operation. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string instances: description: Specifies the desired storage size of the instance diff --git a/deploy/helm/crds/operations.kubeblocks.io_opsrequests.yaml b/deploy/helm/crds/operations.kubeblocks.io_opsrequests.yaml index 2cdeda3128d..9be03754889 100755 --- a/deploy/helm/crds/operations.kubeblocks.io_opsrequests.yaml +++ b/deploy/helm/crds/operations.kubeblocks.io_opsrequests.yaml @@ -176,7 +176,8 @@ spec: items: properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string parameters: description: Specifies the parameters that match the schema @@ -562,7 +563,8 @@ spec: scaling operation. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string scaleIn: description: |- @@ -4046,7 +4048,8 @@ spec: - Logical backups (e.g., 'mysqldump' for MySQL) are unsupported in the current version. type: string componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string inPlace: description: |- @@ -4223,7 +4226,8 @@ spec: configuration. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string configurations: description: |- @@ -4323,7 +4327,8 @@ spec: on. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string required: - componentName @@ -4533,7 +4538,8 @@ spec: on. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string required: - componentName @@ -4554,7 +4560,8 @@ spec: on. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string required: - componentName @@ -4579,7 +4586,11 @@ spec: Refer to ComponentDefinition's Swtichover lifecycle action for more details. type: string componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec. + type: string + componentObjectName: + description: Specifies the name of the Component object. type: string instanceName: description: |- @@ -4587,13 +4598,13 @@ spec: in a consensus system. type: string required: - - componentName - instanceName type: object + x-kubernetes-validations: + - message: need to specified only componentName or componentObjectName + rule: (has(self.componentName) && !has(self.componentObjectName)) + || (!has(self.componentName) && has(self.componentObjectName)) type: array - x-kubernetes-list-map-keys: - - componentName - x-kubernetes-list-type: map x-kubernetes-validations: - message: forbidden to update spec.switchover rule: self == oldSelf @@ -4666,7 +4677,8 @@ spec: maxLength: 64 type: string componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string serviceVersion: description: |- @@ -4727,7 +4739,8 @@ spec: - name x-kubernetes-list-type: map componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string instances: description: Specifies the desired compute resources of the @@ -4839,7 +4852,8 @@ spec: for a volume expansion operation. properties: componentName: - description: Specifies the name of the Component. + description: Specifies the name of the Component as defined + in the cluster.spec type: string instances: description: Specifies the desired storage size of the instance diff --git a/pkg/operations/switchover.go b/pkg/operations/switchover.go index aec7fa45b2c..01dea62c438 100644 --- a/pkg/operations/switchover.go +++ b/pkg/operations/switchover.go @@ -110,18 +110,17 @@ func switchoverPreCheck(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes } for _, switchover := range switchoverList { - compSpec := opsRes.Cluster.Spec.GetComponentByName(switchover.ComponentName) - synthesizedComp, err := buildSynthesizedComp(reqCtx.Ctx, cli, opsRes, compSpec) + synthesizedComp, err := buildSynthesizedComp(reqCtx.Ctx, cli, opsRes, switchover) if err != nil { return err } - + compName := switchover.GetComponentName() if synthesizedComp.LifecycleActions == nil || synthesizedComp.LifecycleActions.Switchover == nil { - return intctrlutil.NewFatalError(fmt.Sprintf(`the component "%s" does not define switchover lifecycle action`, switchover.ComponentName)) + return intctrlutil.NewFatalError(fmt.Sprintf(`the component "%s" does not define switchover lifecycle action`, compName)) } if len(synthesizedComp.Roles) == 0 { - return intctrlutil.NewFatalError(fmt.Sprintf(`the component "%s" does not have any role`, switchover.ComponentName)) + return intctrlutil.NewFatalError(fmt.Sprintf(`the component "%s" does not have any role`, compName)) } getPod := func(name string) (*corev1.Pod, error) { @@ -136,8 +135,8 @@ func switchoverPreCheck(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes } checkOwnership := func(pod *corev1.Pod) error { - if pod.Labels[constant.AppInstanceLabelKey] != synthesizedComp.ClusterName || component.GetComponentNameFromObj(pod) != switchover.ComponentName { - return intctrlutil.NewFatalError(fmt.Sprintf(`the pod "%s" not belongs to the component "%s"`, switchover.InstanceName, switchover.ComponentName)) + if pod.Labels[constant.AppInstanceLabelKey] != synthesizedComp.ClusterName || component.GetComponentNameFromObj(pod) != synthesizedComp.Name { + return intctrlutil.NewFatalError(fmt.Sprintf(`the pod "%s" not belongs to the component "%s"`, switchover.InstanceName, compName)) } return nil } @@ -208,36 +207,36 @@ func handleSwitchover(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes * if switchoverCondition == nil { return errors.New("switchover condition is nil") } - + compName := switchover.GetComponentName() detail := opsv1alpha1.ProgressStatusDetail{ - ObjectKey: getProgressObjectKey(KBSwitchoverKey, switchover.ComponentName), + ObjectKey: getProgressObjectKey(KBSwitchoverKey, compName), Status: opsv1alpha1.ProcessingProgressStatus, - Message: fmt.Sprintf("do switchover for component %s", switchover.ComponentName), + Message: fmt.Sprintf("do switchover for component %s", compName), } - synthesizedComp, err := buildSynthesizedComp(reqCtx.Ctx, cli, opsRes, opsRes.Cluster.Spec.GetComponentByName(switchover.ComponentName)) + synthesizedComp, err := buildSynthesizedComp(reqCtx.Ctx, cli, opsRes, *switchover) if err != nil { - return handleError(reqCtx, opsRequest, &detail, switchover.ComponentName, fmt.Sprintf("build synthesizedComponent failed: %s", err.Error()), failedCount) + return handleError(reqCtx, opsRequest, &detail, compName, fmt.Sprintf("build synthesizedComponent failed: %s", err.Error()), failedCount) } compDef, err := component.GetCompDefByName(reqCtx.Ctx, cli, synthesizedComp.CompDefName) if err != nil { - return handleError(reqCtx, opsRequest, &detail, switchover.ComponentName, fmt.Sprintf("get component definition failed: %s", err.Error()), failedCount) + return handleError(reqCtx, opsRequest, &detail, compName, fmt.Sprintf("get component definition failed: %s", err.Error()), failedCount) } synthesizedComp.TemplateVars, _, err = component.ResolveTemplateNEnvVars(reqCtx.Ctx, cli, synthesizedComp, compDef.Spec.Vars) if err != nil { - return handleError(reqCtx, opsRequest, &detail, switchover.ComponentName, fmt.Sprintf("build synthesizedComponent template vars failed: %s", err.Error()), failedCount) + return handleError(reqCtx, opsRequest, &detail, compName, fmt.Sprintf("build synthesizedComponent template vars failed: %s", err.Error()), failedCount) } if err = doSwitchover(reqCtx.Ctx, cli, synthesizedComp, switchover); err != nil { - return handleError(reqCtx, opsRequest, &detail, switchover.ComponentName, fmt.Sprintf("call switchover action failed: %s", err.Error()), failedCount) + return handleError(reqCtx, opsRequest, &detail, compName, fmt.Sprintf("call switchover action failed: %s", err.Error()), failedCount) } *completedCount++ - detail.Message = fmt.Sprintf("do switchover for component %s succeeded", switchover.ComponentName) + detail.Message = fmt.Sprintf("do switchover for component %s succeeded", compName) detail.Status = opsv1alpha1.SucceedProgressStatus - setComponentSwitchoverProgressDetails(reqCtx.Recorder, opsRequest, appsv1.RunningComponentPhase, detail, switchover.ComponentName) + setComponentSwitchoverProgressDetails(reqCtx.Recorder, opsRequest, appsv1.RunningComponentPhase, detail, compName) return nil } @@ -281,9 +280,50 @@ func setComponentSwitchoverProgressDetails(recorder record.EventRecorder, } } -func buildSynthesizedComp(ctx context.Context, cli client.Client, opsRes *OpsResource, clusterCompSpec *appsv1.ClusterComponentSpec) (*component.SynthesizedComponent, error) { +func getClusterCompSpec(cluster *appsv1.Cluster, clusterCompName string) (*appsv1.ClusterComponentSpec, error) { + compSpec := cluster.Spec.GetComponentByName(clusterCompName) + if compSpec != nil { + return compSpec, nil + } + shardingSpec := cluster.Spec.GetShardingByName(clusterCompName) + if shardingSpec != nil { + return &shardingSpec.Template, nil + } + return nil, fmt.Errorf(`component "%s" not found`, clusterCompName) +} + +func getCompSpecBySwitchover(ctx context.Context, cli client.Client, cluster *appsv1.Cluster, switchover opsv1alpha1.Switchover) (*appsv1.ClusterComponentSpec, error) { + if len(switchover.ComponentName) > 0 { + return getClusterCompSpec(cluster, switchover.ComponentName) + } + compObj := &appsv1.Component{} + if err := cli.Get(ctx, client.ObjectKey{ + Namespace: cluster.Namespace, + Name: switchover.ComponentObjectName, + }, compObj); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf(`component object "%s" not found`, switchover.ComponentObjectName) + } + return nil, err + } + clusterCompName := compObj.Labels[constant.KBAppShardingNameLabelKey] + if len(clusterCompName) == 0 { + clusterCompName = compObj.Labels[constant.KBAppComponentLabelKey] + } + return getClusterCompSpec(cluster, clusterCompName) +} + +func buildSynthesizedComp(ctx context.Context, cli client.Client, opsRes *OpsResource, switchover opsv1alpha1.Switchover) (*component.SynthesizedComponent, error) { + clusterCompSpec, err := getCompSpecBySwitchover(ctx, cli, opsRes.Cluster, switchover) + if err != nil { + return nil, err + } + componentObjectName := constant.GenerateClusterComponentName(opsRes.Cluster.Name, clusterCompSpec.Name) + if len(switchover.ComponentObjectName) > 0 { + componentObjectName = switchover.ComponentObjectName + } compObj, compDefObj, err := component.GetCompNCompDefByName(ctx, cli, - opsRes.Cluster.Namespace, constant.GenerateClusterComponentName(opsRes.Cluster.Name, clusterCompSpec.Name)) + opsRes.Cluster.Namespace, componentObjectName) if err != nil { return nil, err } @@ -291,10 +331,19 @@ func buildSynthesizedComp(ctx context.Context, cli client.Client, opsRes *OpsRes return component.BuildSynthesizedComponent(ctx, cli, compDefObj, compObj, opsRes.Cluster) } -func handleError(reqCtx intctrlutil.RequestCtx, opsRequest *opsv1alpha1.OpsRequest, detail *opsv1alpha1.ProgressStatusDetail, componentName, errorMsg string, failedCount *int32) error { - *failedCount++ +func handleError(reqCtx intctrlutil.RequestCtx, + opsRequest *opsv1alpha1.OpsRequest, + detail *opsv1alpha1.ProgressStatusDetail, + componentName, errorMsg string, + failedCount *int32) error { detail.Message = fmt.Sprintf("component %s %s", componentName, errorMsg) - detail.Status = opsv1alpha1.FailedProgressStatus + detail.Status = opsv1alpha1.ProcessingProgressStatus setComponentSwitchoverProgressDetails(reqCtx.Recorder, opsRequest, appsv1.UpdatingComponentPhase, *detail, componentName) + // set timeout to 5 minutes for handling the switchover + if !detail.StartTime.IsZero() && time.Now().After(detail.StartTime.Add(5*time.Minute)) { + detail.Status = opsv1alpha1.FailedProgressStatus + *failedCount++ + return intctrlutil.NewFatalError("timeout exit: " + errorMsg) + } return nil } diff --git a/pkg/operations/switchover_test.go b/pkg/operations/switchover_test.go index e9c9b1ef3b0..b8093e8b34c 100644 --- a/pkg/operations/switchover_test.go +++ b/pkg/operations/switchover_test.go @@ -160,8 +160,8 @@ var _ = Describe("", func() { instanceName := fmt.Sprintf("%s-%s-%d", clusterObj.Name, defaultCompName, 1) ops.Spec.SwitchoverList = []opsv1alpha1.Switchover{ { - ComponentOps: opsv1alpha1.ComponentOps{ComponentName: defaultCompName}, - InstanceName: instanceName, + ComponentName: defaultCompName, + InstanceName: instanceName, }, } opsRes.OpsRequest = testops.CreateOpsRequest(ctx, testCtx, ops) @@ -199,7 +199,7 @@ var _ = Describe("", func() { candidateName := fmt.Sprintf("%s-%s-%d", clusterObj.Name, defaultCompName, 0) ops.Spec.SwitchoverList = []opsv1alpha1.Switchover{ { - ComponentOps: opsv1alpha1.ComponentOps{ComponentName: defaultCompName}, + ComponentName: defaultCompName, InstanceName: instanceName, CandidateName: candidateName, }, From 52286f71694a731ec5842d832209683425a02f43 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Fri, 10 Jan 2025 18:05:05 +0800 Subject: [PATCH 2/2] fix bug --- pkg/operations/switchover_test.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/operations/switchover_test.go b/pkg/operations/switchover_test.go index b8093e8b34c..920dfccdca2 100644 --- a/pkg/operations/switchover_test.go +++ b/pkg/operations/switchover_test.go @@ -46,6 +46,7 @@ var _ = Describe("", func() { compDefName = "test-compdef-" clusterName = "test-cluster-" compDefObj *appsv1.ComponentDefinition + compObj *appsv1.Component clusterObj *appsv1.Cluster ) @@ -100,9 +101,10 @@ var _ = Describe("", func() { Create(&testCtx).GetObject() By("creating a component") - _ = testapps.NewComponentFactory(testCtx.DefaultNamespace, clusterObj.Name+"-"+defaultCompName, compDefObj.Name). + compObj = testapps.NewComponentFactory(testCtx.DefaultNamespace, clusterObj.Name+"-"+defaultCompName, compDefObj.Name). AddAppManagedByLabel(). AddAppInstanceLabel(clusterObj.Name). + AddAppComponentLabel(defaultCompName). AddAnnotations(constant.KBAppClusterUIDKey, string(clusterObj.UID)). Create(&testCtx). GetObject() @@ -191,7 +193,7 @@ var _ = Describe("", func() { Expect(err).ShouldNot(HaveOccurred()) }) - It("Test switchover OpsRequest with candidate", func() { + testSwitchoverWithCandidate := func(useComponentObjectName bool) { By("create switchover opsRequest") ops := testops.NewOpsRequestObj("ops-switchover-"+testCtx.GetRandomStr(), testCtx.DefaultNamespace, clusterObj.Name, opsv1alpha1.SwitchoverType) @@ -204,6 +206,11 @@ var _ = Describe("", func() { CandidateName: candidateName, }, } + if useComponentObjectName { + ops.Spec.SwitchoverList[0].ComponentName = "" + ops.Spec.SwitchoverList[0].ComponentObjectName = compObj.Name + } + fmt.Printf("ops: %#v\n", ops.Spec.SwitchoverList[0]) opsRes.OpsRequest = testops.CreateOpsRequest(ctx, testCtx, ops) opsRes.OpsRequest.Status.Phase = opsv1alpha1.OpsPendingPhase @@ -230,6 +237,14 @@ var _ = Describe("", func() { By("do reconcile switchover action") _, err = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) Expect(err).ShouldNot(HaveOccurred()) + } + + It("Test switchover OpsRequest with candidate", func() { + testSwitchoverWithCandidate(false) + }) + + It("Test switchover OpsRequest with candidate and specified a component object name", func() { + testSwitchoverWithCandidate(true) }) }) })