From a7e02cde6fb6df135398c5ba3375b4479084f04e Mon Sep 17 00:00:00 2001 From: free6om Date: Wed, 24 Apr 2024 11:50:53 +0800 Subject: [PATCH] feat: support KB_POD_LIST env --- apis/apps/v1alpha1/cluster_types.go | 8 +++++ apis/workloads/v1alpha1/instanceset_types.go | 8 +++++ .../apps/transformer_component_service.go | 22 +++---------- pkg/controller/instanceset/instance_util.go | 33 ++++++++++++++++++- .../instanceset/instance_util_test.go | 26 +++++++++++++-- pkg/controller/instanceset/object_builder.go | 11 +++++++ .../reconciler_instance_alignment_test.go | 14 ++++---- .../instanceset/reconciler_update_test.go | 20 +++++------ 8 files changed, 104 insertions(+), 38 deletions(-) diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index 7ca245b1575..a73fb90146e 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -1538,6 +1538,14 @@ func (r *ClusterComponentSpec) ToVolumeClaimTemplates() []corev1.PersistentVolum return ts } +func (t *InstanceTemplate) GetName() string { + return t.Name +} + +func (t *InstanceTemplate) GetReplicas() *int32 { + return t.Replicas +} + // GetClusterUpRunningPhases returns Cluster running or partially running phases. func GetClusterUpRunningPhases() []ClusterPhase { return []ClusterPhase{ diff --git a/apis/workloads/v1alpha1/instanceset_types.go b/apis/workloads/v1alpha1/instanceset_types.go index 1bc31586928..80c37f02d3a 100644 --- a/apis/workloads/v1alpha1/instanceset_types.go +++ b/apis/workloads/v1alpha1/instanceset_types.go @@ -575,6 +575,14 @@ type MemberStatus struct { ReadyWithoutPrimary bool `json:"readyWithoutPrimary"` } +func (t *InstanceTemplate) GetName() string { + return t.Name +} + +func (t *InstanceTemplate) GetReplicas() *int32 { + return t.Replicas +} + func init() { SchemeBuilder.Register(&InstanceSet{}, &InstanceSetList{}) } diff --git a/controllers/apps/transformer_component_service.go b/controllers/apps/transformer_component_service.go index 271e43f6b14..fe68a9dc275 100644 --- a/controllers/apps/transformer_component_service.go +++ b/controllers/apps/transformer_component_service.go @@ -37,7 +37,6 @@ import ( "github.com/apecloud/kubeblocks/pkg/controller/graph" "github.com/apecloud/kubeblocks/pkg/controller/instanceset" "github.com/apecloud/kubeblocks/pkg/controller/model" - intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" ) var ( @@ -210,23 +209,10 @@ func (t *componentServiceTransformer) skipDefaultHeadlessSvc(synthesizeComp *com } func generatePodNames(synthesizeComp *component.SynthesizedComponent) []string { - - templateReplicasCnt := int32(0) - for _, template := range synthesizeComp.Instances { - if len(template.Name) > 0 { - templateReplicasCnt += intctrlutil.TemplateReplicas(template) - } - } - - podNames := make([]string, 0) workloadName := constant.GenerateWorkloadNamePattern(synthesizeComp.ClusterName, synthesizeComp.Name) - for _, template := range synthesizeComp.Instances { - templateNames := instanceset.GenerateInstanceNamesFromTemplate(workloadName, template.Name, intctrlutil.TemplateReplicas(template), synthesizeComp.OfflineInstances) - podNames = append(podNames, templateNames...) - } - if templateReplicasCnt < synthesizeComp.Replicas { - names := instanceset.GenerateInstanceNamesFromTemplate(workloadName, "", synthesizeComp.Replicas-templateReplicasCnt, synthesizeComp.OfflineInstances) - podNames = append(podNames, names...) + var templates []instanceset.InstanceTemplate + for i := range synthesizeComp.Instances { + templates = append(templates, &synthesizeComp.Instances[i]) } - return podNames + return instanceset.GenerateAllInstanceNames(workloadName, synthesizeComp.Replicas, templates, synthesizeComp.OfflineInstances) } diff --git a/pkg/controller/instanceset/instance_util.go b/pkg/controller/instanceset/instance_util.go index 76d973561e5..18db495c566 100644 --- a/pkg/controller/instanceset/instance_util.go +++ b/pkg/controller/instanceset/instance_util.go @@ -45,6 +45,11 @@ import ( "github.com/apecloud/kubeblocks/pkg/controller/model" ) +type InstanceTemplate interface { + GetName() string + GetReplicas() *int32 +} + type instanceTemplateExt struct { Name string Replicas int32 @@ -125,7 +130,7 @@ func baseSort(x any, getNameNOrdinalFunc func(i int) (string, int), getRolePrior name1, ordinal1 := getNameNOrdinalFunc(i) name2, ordinal2 := getNameNOrdinalFunc(j) if name1 != name2 { - return name1 < name2 + return name1 > name2 } return ordinal1 > ordinal2 }) @@ -210,6 +215,32 @@ func buildInstanceName2TemplateMap(itsExt *instanceSetExt) (map[string]*instance return allNameTemplateMap, nil } +func GenerateAllInstanceNames(parentName string, replicas int32, templates []InstanceTemplate, offlineInstances []string) []string { + templateReplicas := func(template InstanceTemplate) int32 { + if template.GetReplicas() != nil { + return *template.GetReplicas() + } + return 1 + } + totalReplicas := int32(0) + instanceNameList := make([]string, 0) + for _, template := range templates { + replicas := templateReplicas(template) + names := GenerateInstanceNamesFromTemplate(parentName, template.GetName(), replicas, offlineInstances) + instanceNameList = append(instanceNameList, names...) + totalReplicas += replicas + } + if totalReplicas < replicas { + names := GenerateInstanceNamesFromTemplate(parentName, "", replicas-totalReplicas, offlineInstances) + instanceNameList = append(instanceNameList, names...) + } + getNameNOrdinalFunc := func(i int) (string, int) { + return ParseParentNameAndOrdinal(instanceNameList[i]) + } + baseSort(instanceNameList, getNameNOrdinalFunc, nil, true) + return instanceNameList +} + func GenerateInstanceNamesFromTemplate(parentName, templateName string, replicas int32, offlineInstances []string) []string { instanceNames, _ := GenerateInstanceNames(parentName, templateName, replicas, 0, offlineInstances) return instanceNames diff --git a/pkg/controller/instanceset/instance_util_test.go b/pkg/controller/instanceset/instance_util_test.go index 0a3458cd6fc..41a16e0d5b2 100644 --- a/pkg/controller/instanceset/instance_util_test.go +++ b/pkg/controller/instanceset/instance_util_test.go @@ -28,6 +28,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" workloads "github.com/apecloud/kubeblocks/apis/workloads/v1alpha1" @@ -61,7 +62,7 @@ var _ = Describe("instance util test", func() { builder.NewPodBuilder(namespace, "pod-10").AddLabels(RoleLabelKey, "learner").GetObject(), builder.NewPodBuilder(namespace, "foo-20").AddLabels(RoleLabelKey, "learner").GetObject(), } - expectedOrder := []string{"pod-4", "pod-2", "foo-20", "pod-10", "pod-6", "pod-3", "pod-1", "pod-0", "pod-5"} + expectedOrder := []string{"pod-4", "pod-2", "pod-10", "pod-6", "pod-3", "foo-20", "pod-1", "pod-0", "pod-5"} sortObjects(pods, priorityMap, false) for i, pod := range pods { @@ -413,7 +414,28 @@ var _ = Describe("instance util test", func() { return ParseParentNameAndOrdinal(instanceNameList[i]) } baseSort(instanceNameList, getNameNOrdinalFunc, nil, true) - podNamesExpected := []string{"foo-bar-0", "foo-bar-2", "foo-1", "foo-2"} + podNamesExpected := []string{"foo-1", "foo-2", "foo-bar-0", "foo-bar-2"} + Expect(instanceNameList).Should(Equal(podNamesExpected)) + }) + }) + + Context("GenerateAllInstanceNames", func() { + It("should work well", func() { + parentName := "foo" + templatesFoo := &workloads.InstanceTemplate{ + Name: "foo", + Replicas: pointer.Int32(1), + } + templateBar := &workloads.InstanceTemplate{ + Name: "bar", + Replicas: pointer.Int32(2), + } + var templates []InstanceTemplate + templates = append(templates, templatesFoo, templateBar) + offlineInstances := []string{"foo-bar-1", "foo-0"} + instanceNameList := GenerateAllInstanceNames(parentName, 5, templates, offlineInstances) + + podNamesExpected := []string{"foo-1", "foo-2", "foo-bar-0", "foo-bar-2", "foo-foo-0"} Expect(instanceNameList).Should(Equal(podNamesExpected)) }) }) diff --git a/pkg/controller/instanceset/object_builder.go b/pkg/controller/instanceset/object_builder.go index 7d9454d541e..7d5ef98a6f4 100644 --- a/pkg/controller/instanceset/object_builder.go +++ b/pkg/controller/instanceset/object_builder.go @@ -532,6 +532,14 @@ func buildEnvConfigData(its workloads.InstanceSet) map[string]string { envData[prefix+"FOLLOWERS"] = followers } } + // generate all pod names + generatePodNames := func() []string { + var instances []InstanceTemplate + for i := range its.Spec.Instances { + instances = append(instances, &its.Spec.Instances[i]) + } + return GenerateAllInstanceNames(its.Name, *its.Spec.Replicas, instances, its.Spec.OfflineInstances) + } prefix := constant.KBPrefix + "_ITS_" envData[prefix+"N"] = strReplicas @@ -546,6 +554,9 @@ func buildEnvConfigData(its workloads.InstanceSet) map[string]string { envData[prefix+"REPLICA_COUNT"] = strReplicas generateReplicaEnv(prefix) generateMemberEnv(prefix) + // KB_POD_LIST + names := generatePodNames() + envData[prefix+"POD_LIST"] = strings.Join(names, ",") // have backward compatible handling for CM key with 'compDefName' being part of the key name, prior 0.5.0 // and introduce env/cm key naming reference complexity diff --git a/pkg/controller/instanceset/reconciler_instance_alignment_test.go b/pkg/controller/instanceset/reconciler_instance_alignment_test.go index c0c1929997d..4e3fed325fc 100644 --- a/pkg/controller/instanceset/reconciler_instance_alignment_test.go +++ b/pkg/controller/instanceset/reconciler_instance_alignment_test.go @@ -54,8 +54,8 @@ var _ = Describe("replicas alignment reconciler test", func() { Expect(reconciler.PreCondition(tree)).Should(Equal(kubebuilderx.ResultSatisfied)) By("prepare current tree") - // desired: bar-hello-0, bar-foo-0, bar-foo-1, bar-0, bar-1, bar-2, bar-3 - // current: bar-foo-0, bar-1 + // desired: bar-0, bar-1, bar-2, bar-3, bar-foo-0, bar-foo-1, bar-hello-0 + // current: bar-1, bar-foo-0 replicas := int32(7) its.Spec.Replicas = &replicas nameHello := "hello" @@ -84,7 +84,7 @@ var _ = Describe("replicas alignment reconciler test", func() { Expect(err).Should(BeNil()) newTree, err := reconciler.Reconcile(orderedReadyTree) Expect(err).Should(BeNil()) - // desired: bar-hello-0, bar-foo-0, bar-1 + // desired: bar-0, bar-1, bar-foo-0 pods := newTree.List(&corev1.Pod{}) Expect(pods).Should(HaveLen(3)) currentPodSnapshot := make(model.ObjectSnapshot) @@ -93,8 +93,8 @@ var _ = Describe("replicas alignment reconciler test", func() { Expect(err).Should(BeNil()) currentPodSnapshot[*name] = object } - podHelloBar0 := builder.NewPodBuilder(namespace, "bar-hello-0").GetObject() - for _, object := range []client.Object{podFoo0, podHelloBar0, podBar1} { + podBar0 := builder.NewPodBuilder(namespace, "bar-0").GetObject() + for _, object := range []client.Object{podFoo0, podBar0, podBar1} { name, err := model.GetGVKName(object) Expect(err).Should(BeNil()) _, ok := currentPodSnapshot[*name] @@ -109,7 +109,7 @@ var _ = Describe("replicas alignment reconciler test", func() { parallelITS.Spec.PodManagementPolicy = appsv1.ParallelPodManagement newTree, err = reconciler.Reconcile(parallelTree) Expect(err).Should(BeNil()) - // desired: bar-hello-0, bar-foo-0, bar-foo-1, bar-0, bar-1, bar-2, bar-3 + // desired: bar-0, bar-1, bar-2, bar-3, bar-foo-0, bar-foo-1, bar-hello-0 pods = newTree.List(&corev1.Pod{}) Expect(pods).Should(HaveLen(7)) currentPodSnapshot = make(model.ObjectSnapshot) @@ -123,7 +123,7 @@ var _ = Describe("replicas alignment reconciler test", func() { podFoo1 := builder.NewPodBuilder(namespace, its.Name+"-foo-1").GetObject() podBar2 := builder.NewPodBuilder(namespace, "bar-2").GetObject() podBar3 := builder.NewPodBuilder(namespace, "bar-3").GetObject() - for _, object := range []client.Object{podHello, podFoo0, podFoo1, podHelloBar0, podBar1, podBar2, podBar3} { + for _, object := range []client.Object{podHello, podFoo0, podFoo1, podBar0, podBar1, podBar2, podBar3} { name, err := model.GetGVKName(object) Expect(err).Should(BeNil()) _, ok := currentPodSnapshot[*name] diff --git a/pkg/controller/instanceset/reconciler_update_test.go b/pkg/controller/instanceset/reconciler_update_test.go index fdecde32b18..4624f525462 100644 --- a/pkg/controller/instanceset/reconciler_update_test.go +++ b/pkg/controller/instanceset/reconciler_update_test.go @@ -60,7 +60,7 @@ var _ = Describe("update reconciler test", func() { Expect(reconciler.PreCondition(tree)).Should(Equal(kubebuilderx.ResultSatisfied)) By("prepare current tree") - // desired: bar-3, bar-2, bar-1, bar-0, bar-foo-1, bar-foo-0, bar-hello-0 + // desired: bar-hello-0, bar-foo-1, bar-foo-0, bar-3, bar-2, bar-1, bar-0 replicas := int32(7) its.Spec.Replicas = &replicas its.Spec.PodManagementPolicy = appsv1.ParallelPodManagement @@ -137,13 +137,13 @@ var _ = Describe("update reconciler test", func() { reconciler = NewUpdateReconciler() By("reconcile with default UpdateStrategy(RollingUpdate, no partition, MaxUnavailable=1)") - // order: bar-3, bar-2, bar-1, bar-0, bar-foo-1, bar-foo-0, bar-hello-0 - // expected: bar-3 being deleted + // order: bar-hello-0, bar-foo-1, bar-foo-0, bar-3, bar-2, bar-1, bar-0 + // expected: bar-hello-0 being deleted defaultTree, err := newTree.DeepCopy() Expect(err).Should(BeNil()) _, err = reconciler.Reconcile(defaultTree) Expect(err).Should(BeNil()) - expectUpdatedPods(defaultTree, []string{"bar-3"}) + expectUpdatedPods(defaultTree, []string{"bar-hello-0"}) By("reconcile with Partition=50% and MaxUnavailable=2") partitionTree, err := newTree.DeepCopy() @@ -158,13 +158,13 @@ var _ = Describe("update reconciler test", func() { MaxUnavailable: &maxUnavailable, }, } - // order: bar-3, bar-2, bar-1, bar-0, bar-foo-1, bar-foo-0, bar-hello-0 - // expected: bar-0, bar-1 being deleted + // order: bar-hello-0, bar-foo-1, bar-foo-0, bar-3, bar-2, bar-1, bar-0 + // expected: bar-hello-0, bar-foo-1 being deleted _, err = reconciler.Reconcile(partitionTree) Expect(err).Should(BeNil()) - expectUpdatedPods(partitionTree, []string{"bar-3", "bar-2"}) + expectUpdatedPods(partitionTree, []string{"bar-hello-0", "bar-foo-1"}) - By("update 'bar-0' revision to the updated value") + By("update revisions to the updated value") partitionTree, err = newTree.DeepCopy() Expect(err).Should(BeNil()) root, ok = partitionTree.GetRoot().(*workloads.InstanceSet) @@ -175,7 +175,7 @@ var _ = Describe("update reconciler test", func() { MaxUnavailable: &maxUnavailable, }, } - for _, name := range []string{"bar-3", "bar-2"} { + for _, name := range []string{"bar-hello-0", "bar-foo-1"} { pod := builder.NewPodBuilder(namespace, name).GetObject() object, err := partitionTree.Get(pod) Expect(err).Should(BeNil()) @@ -185,7 +185,7 @@ var _ = Describe("update reconciler test", func() { } _, err = reconciler.Reconcile(partitionTree) Expect(err).Should(BeNil()) - expectUpdatedPods(partitionTree, []string{"bar-1"}) + expectUpdatedPods(partitionTree, []string{"bar-foo-0"}) By("reconcile with UpdateStrategy='OnDelete'") onDeleteTree, err := newTree.DeepCopy()