K8SPSMDB-227: Add topologySpreadConstraints (#1280)

* K8SPSMDB-227: Add `topologySpreadConstraints` https://jira.percona.com/browse/K8SPSMDB-227 * fix tests * fix defaults * fix `upgrade-consistency` test * update `cr.yaml` * Revert "fix tests" This reverts commit 6796ce7. * don't add topologySpreadConstraints by default * fix `TestSetSafeDefault` --------- Co-authored-by: Inel Pandzic <[email protected]>
percona · Sep 1, 2023 · ef110e3 · ef110e3
1 parent 06ffb2d
commit ef110e3
Show file tree

Hide file tree

Showing 12 changed files with 1,903 additions and 17 deletions.
diff --git a/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml b/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml
diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml
diff --git a/deploy/cr.yaml b/deploy/cr.yaml
@@ -49,6 +49,13 @@ spec:
   - name: rs0
     size: 3
 #    terminationGracePeriodSeconds: 300
+#    topologySpreadConstraints:
+#      - labelSelector:
+#          matchLabels:
+#            app.kubernetes.io/name: percona-server-mongodb
+#        maxSkew: 1
+#        topologyKey: kubernetes.io/hostname
+#        whenUnsatisfiable: DoNotSchedule
 #    externalNodes:
 #    - host: 34.124.76.90
 #    - host: 34.124.76.91
@@ -310,6 +317,13 @@ spec:
     configsvrReplSet:
       size: 3
 #      terminationGracePeriodSeconds: 300
+#      topologySpreadConstraints:
+#        - labelSelector:
+#            matchLabels:
+#              app.kubernetes.io/name: percona-server-mongodb
+#          maxSkew: 1
+#          topologyKey: kubernetes.io/hostname
+#          whenUnsatisfiable: DoNotSchedule
 #      externalNodes:
 #      - host: 34.124.76.93
 #      - host: 34.124.76.94
@@ -407,6 +421,13 @@ spec:
     mongos:
       size: 3
 #      terminationGracePeriodSeconds: 300
+#      topologySpreadConstraints:
+#        - labelSelector:
+#            matchLabels:
+#              app.kubernetes.io/name: percona-server-mongodb
+#          maxSkew: 1
+#          topologyKey: kubernetes.io/hostname
+#          whenUnsatisfiable: DoNotSchedule
 #      # for more configuration fields refer to https://docs.mongodb.com/manual/reference/configuration-options/
 #      configuration: |
 #        systemLog:

diff --git a/deploy/crd.yaml b/deploy/crd.yaml
diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml
diff --git a/e2e-tests/version-service/conf/crd.yaml b/e2e-tests/version-service/conf/crd.yaml
diff --git a/pkg/apis/psmdb/v1/psmdb_defaults.go b/pkg/apis/psmdb/v1/psmdb_defaults.go
@@ -273,7 +273,7 @@ func (cr *PerconaServerMongoDB) CheckNSetDefaults(platform version.Platform, log
 			cr.Spec.Sharding.Mongos.ReadinessProbe.FailureThreshold = 3
 		}
 
-		cr.Spec.Sharding.Mongos.reconcileOpts()
+		cr.Spec.Sharding.Mongos.reconcileOpts(cr)
 
 		if err := cr.Spec.Sharding.Mongos.Configuration.SetDefaults(); err != nil {
 			return errors.Wrap(err, "failed to set configuration defaults")
@@ -558,10 +558,10 @@ func (rs *ReplsetSpec) SetDefaults(platform version.Platform, cr *PerconaServerM
 		rs.Expose.ExposeType = corev1.ServiceTypeClusterIP
 	}
 
-	rs.MultiAZ.reconcileOpts()
+	rs.MultiAZ.reconcileOpts(cr)
 
 	if rs.Arbiter.Enabled {
-		rs.Arbiter.MultiAZ.reconcileOpts()
+		rs.Arbiter.MultiAZ.reconcileOpts(cr)
 	}
 
 	if !cr.Spec.UnsafeConf && cr.DeletionTimestamp == nil {
@@ -694,7 +694,7 @@ func (nv *NonVotingSpec) SetDefaults(cr *PerconaServerMongoDB, rs *ReplsetSpec)
 		nv.ServiceAccountName = WorkloadSA
 	}
 
-	nv.MultiAZ.reconcileOpts()
+	nv.MultiAZ.reconcileOpts(cr)
 
 	if nv.ContainerSecurityContext == nil {
 		nv.ContainerSecurityContext = rs.ContainerSecurityContext
@@ -741,8 +741,9 @@ func (rs *ReplsetSpec) setSafeDefaults(log logr.Logger) {
 	}
 }
 
-func (m *MultiAZ) reconcileOpts() {
+func (m *MultiAZ) reconcileOpts(cr *PerconaServerMongoDB) {
 	m.reconcileAffinityOpts()
+	m.reconcileTopologySpreadConstraints(cr)
 
 	if m.PodDisruptionBudget == nil {
 		defaultMaxUnavailable := intstr.FromInt(1)
@@ -786,6 +787,24 @@ func (m *MultiAZ) reconcileAffinityOpts() {
 	}
 }
 
+func (m *MultiAZ) reconcileTopologySpreadConstraints(cr *PerconaServerMongoDB) {
+	if cr.CompareVersion("1.15.0") < 0 {
+		return
+	}
+
+	for i := range m.TopologySpreadConstraints {
+		if m.TopologySpreadConstraints[i].MaxSkew == 0 {
+			m.TopologySpreadConstraints[i].MaxSkew = 1
+		}
+		if m.TopologySpreadConstraints[i].TopologyKey == "" {
+			m.TopologySpreadConstraints[i].TopologyKey = defaultAffinityTopologyKey
+		}
+		if m.TopologySpreadConstraints[i].WhenUnsatisfiable == "" {
+			m.TopologySpreadConstraints[i].WhenUnsatisfiable = corev1.DoNotSchedule
+		}
+	}
+}
+
 func (v *VolumeSpec) reconcileOpts() error {
 	if v.EmptyDir == nil && v.HostPath == nil && v.PersistentVolumeClaim.PersistentVolumeClaimSpec == nil {
 		v.PersistentVolumeClaim.PersistentVolumeClaimSpec = &corev1.PersistentVolumeClaimSpec{}

diff --git a/pkg/apis/psmdb/v1/psmdb_defaults_test.go b/pkg/apis/psmdb/v1/psmdb_defaults_test.go
@@ -226,8 +226,9 @@ func TestSetSafeDefault(t *testing.T) {
 	cr := &api.PerconaServerMongoDB{
 		ObjectMeta: metav1.ObjectMeta{Name: "psmdb-mock", Namespace: "psmdb"},
 		Spec: api.PerconaServerMongoDBSpec{
-			Replsets: []*api.ReplsetSpec{{Name: "rs0", Size: 3}, {Name: "rs1", Size: 3}},
-			Sharding: api.Sharding{Enabled: true, Mongos: &api.MongosSpec{Size: 3}},
+			CRVersion: version.Version,
+			Replsets:  []*api.ReplsetSpec{{Name: "rs0", Size: 3}, {Name: "rs1", Size: 3}},
+			Sharding:  api.Sharding{Enabled: true, Mongos: &api.MongosSpec{Size: 3}},
 		},
 	}
 

diff --git a/pkg/apis/psmdb/v1/psmdb_types.go b/pkg/apis/psmdb/v1/psmdb_types.go
@@ -287,16 +287,17 @@ func (spec *PMMSpec) ShouldUseAPIKeyAuth(secret *corev1.Secret) bool {
 }
 
 type MultiAZ struct {
-	Affinity                      *PodAffinity             `json:"affinity,omitempty"`
-	NodeSelector                  map[string]string        `json:"nodeSelector,omitempty"`
-	Tolerations                   []corev1.Toleration      `json:"tolerations,omitempty"`
-	PriorityClassName             string                   `json:"priorityClassName,omitempty"`
-	ServiceAccountName            string                   `json:"serviceAccountName,omitempty"`
-	Annotations                   map[string]string        `json:"annotations,omitempty"`
-	Labels                        map[string]string        `json:"labels,omitempty"`
-	PodDisruptionBudget           *PodDisruptionBudgetSpec `json:"podDisruptionBudget,omitempty"`
-	TerminationGracePeriodSeconds *int64                   `json:"terminationGracePeriodSeconds,omitempty"`
-	RuntimeClassName              *string                  `json:"runtimeClassName,omitempty"`
+	Affinity                      *PodAffinity                      `json:"affinity,omitempty"`
+	TopologySpreadConstraints     []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
+	NodeSelector                  map[string]string                 `json:"nodeSelector,omitempty"`
+	Tolerations                   []corev1.Toleration               `json:"tolerations,omitempty"`
+	PriorityClassName             string                            `json:"priorityClassName,omitempty"`
+	ServiceAccountName            string                            `json:"serviceAccountName,omitempty"`
+	Annotations                   map[string]string                 `json:"annotations,omitempty"`
+	Labels                        map[string]string                 `json:"labels,omitempty"`
+	PodDisruptionBudget           *PodDisruptionBudgetSpec          `json:"podDisruptionBudget,omitempty"`
+	TerminationGracePeriodSeconds *int64                            `json:"terminationGracePeriodSeconds,omitempty"`
+	RuntimeClassName              *string                           `json:"runtimeClassName,omitempty"`
 
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 

diff --git a/pkg/apis/psmdb/v1/zz_generated.deepcopy.go b/pkg/apis/psmdb/v1/zz_generated.deepcopy.go
diff --git a/pkg/psmdb/mongos.go b/pkg/psmdb/mongos.go
@@ -125,6 +125,7 @@ func MongosTemplateSpec(cr *api.PerconaServerMongoDB, initImage string, log logr
 			HostAliases:                   cr.Spec.Sharding.Mongos.HostAliases,
 			SecurityContext:               cr.Spec.Sharding.Mongos.PodSecurityContext,
 			Affinity:                      PodAffinity(cr, cr.Spec.Sharding.Mongos.MultiAZ.Affinity, ls),
+			TopologySpreadConstraints:     PodTopologySpreadConstraints(cr, cr.Spec.Sharding.Mongos.MultiAZ.TopologySpreadConstraints, ls),
 			NodeSelector:                  cr.Spec.Sharding.Mongos.MultiAZ.NodeSelector,
 			Tolerations:                   cr.Spec.Sharding.Mongos.MultiAZ.Tolerations,
 			TerminationGracePeriodSeconds: cr.Spec.Sharding.Mongos.MultiAZ.TerminationGracePeriodSeconds,

diff --git a/pkg/psmdb/statefulset.go b/pkg/psmdb/statefulset.go
@@ -144,6 +144,7 @@ func StatefulSpec(ctx context.Context, cr *api.PerconaServerMongoDB, replset *ap
 				HostAliases:                   replset.HostAliases,
 				SecurityContext:               podSecurityContext,
 				Affinity:                      PodAffinity(cr, multiAZ.Affinity, customLabels),
+				TopologySpreadConstraints:     PodTopologySpreadConstraints(cr, multiAZ.TopologySpreadConstraints, customLabels),
 				NodeSelector:                  multiAZ.NodeSelector,
 				Tolerations:                   multiAZ.Tolerations,
 				TerminationGracePeriodSeconds: multiAZ.TerminationGracePeriodSeconds,
@@ -228,6 +229,21 @@ func PodAffinity(cr *api.PerconaServerMongoDB, af *api.PodAffinity, labels map[s
 	return nil
 }
 
+func PodTopologySpreadConstraints(cr *api.PerconaServerMongoDB, tscs []corev1.TopologySpreadConstraint, ls map[string]string) []corev1.TopologySpreadConstraint {
+	result := make([]corev1.TopologySpreadConstraint, 0, len(tscs))
+
+	for _, tsc := range tscs {
+		if tsc.LabelSelector == nil && tsc.MatchLabelKeys == nil {
+			tsc.LabelSelector = &metav1.LabelSelector{
+				MatchLabels: ls,
+			}
+		}
+
+		result = append(result, tsc)
+	}
+	return result
+}
+
 func isEncryptionEnabled(cr *api.PerconaServerMongoDB, replset *api.ReplsetSpec) (bool, error) {
 	enabled, err := replset.Configuration.IsEncryptionEnabled()
 	if err != nil {