Add Tolerations to Build and BuildRun objects

Signed-off-by: Dylan Orzel <[email protected]>

.github/workflows/ci.yml

@@ -181,7 +181,7 @@ jobs:
           version: v0.24.0
           node_image: kindest/node:${{ matrix.kubernetes }}
           cluster_name: kind
-          config: test/kind/config.yaml
+          config: test/kind/config_three_node.yaml
           wait: 120s
       - name: Verify kind cluster
         run: |
@@ -190,18 +190,21 @@ jobs:
           echo "# KinD nodes:"
           kubectl get nodes
 
-          NODE_STATUS=$(kubectl get node kind-control-plane -o json | jq -r .'status.conditions[] | select(.type == "Ready") | .status')
-          if [ "${NODE_STATUS}" != "True" ]; then
-            echo "# Node is not ready:"
-            kubectl describe node kind-control-plane
+          for nodename in $(kubectl get nodes -o name); do
+            kubectl wait --for=condition=Ready=true ${nodename} --timeout=60s
+            NODE_STATUS=$(kubectl get ${nodename} -o json | jq -r .'status.conditions[] | select(.type == "Ready") | .status')
+            if [ "${NODE_STATUS}" != "True" ]; then
+              echo "# Node is not ready:"
+              kubectl describe ${nodename}
 
-            echo "# Pods:"
-            kubectl get pod -A
-            echo "# Events:"
-            kubectl get events -A
+              echo "# Pods:"
+              kubectl get pod -A
+              echo "# Events:"
+              kubectl get events -A
 
-            exit 1
-          fi
+              exit 1
+            fi
+          done
       - name: Install Tekton
         env:
           TEKTON_VERSION: ${{ matrix.tekton }}

deploy/crds/shipwright.io_buildruns.yaml

@@ -7535,6 +7535,45 @@ spec:
                           Build should take to execute.
                         format: duration
                         type: string
+                      tolerations:
+                        description: If specified, the pod's tolerations.
+                        items:
+                          description: |-
+                            The pod this Toleration is attached to tolerates any taint that matches
+                            the triple <key,value,effect> using the matching operator <operator>.
+                          properties:
+                            effect:
+                              description: |-
+                                Effect indicates the taint effect to match. Empty means match all taint effects.
+                                When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                              type: string
+                            key:
+                              description: |-
+                                Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                                If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                              type: string
+                            operator:
+                              description: |-
+                                Operator represents a key's relationship to the value.
+                                Valid operators are Exists and Equal. Defaults to Equal.
+                                Exists is equivalent to wildcard for value, so that a pod can
+                                tolerate all taints of a particular category.
+                              type: string
+                            tolerationSeconds:
+                              description: |-
+                                TolerationSeconds represents the period of time the toleration (which must be
+                                of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                                it is not set, which means tolerate the taint forever (do not evict). Zero and
+                                negative values will be treated as 0 (evict immediately) by the system.
+                              format: int64
+                              type: integer
+                            value:
+                              description: |-
+                                Value is the taint value the toleration matches to.
+                                If the operator is Exists, the value should be empty, otherwise just a regular string.
+                              type: string
+                          type: object
+                        type: array
                       trigger:
                         description: Trigger defines the scenarios where a new build
                           should be triggered.
@@ -9753,6 +9792,45 @@ spec:
                 description: Timeout defines the maximum run time of this BuildRun.
                 format: duration
                 type: string
+              tolerations:
+                description: If specified, the pod's tolerations.
+                items:
+                  description: |-
+                    The pod this Toleration is attached to tolerates any taint that matches
+                    the triple <key,value,effect> using the matching operator <operator>.
+                  properties:
+                    effect:
+                      description: |-
+                        Effect indicates the taint effect to match. Empty means match all taint effects.
+                        When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                      type: string
+                    key:
+                      description: |-
+                        Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                        If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                      type: string
+                    operator:
+                      description: |-
+                        Operator represents a key's relationship to the value.
+                        Valid operators are Exists and Equal. Defaults to Equal.
+                        Exists is equivalent to wildcard for value, so that a pod can
+                        tolerate all taints of a particular category.
+                      type: string
+                    tolerationSeconds:
+                      description: |-
+                        TolerationSeconds represents the period of time the toleration (which must be
+                        of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                        it is not set, which means tolerate the taint forever (do not evict). Zero and
+                        negative values will be treated as 0 (evict immediately) by the system.
+                      format: int64
+                      type: integer
+                    value:
+                      description: |-
+                        Value is the taint value the toleration matches to.
+                        If the operator is Exists, the value should be empty, otherwise just a regular string.
+                      type: string
+                  type: object
+                type: array
               volumes:
                 description: |-
                   Volumes contains volume Overrides of the BuildStrategy volumes in case those are allowed
@@ -11959,6 +12037,45 @@ spec:
                       should take to execute.
                     format: duration
                     type: string
+                  tolerations:
+                    description: If specified, the pod's tolerations.
+                    items:
+                      description: |-
+                        The pod this Toleration is attached to tolerates any taint that matches
+                        the triple <key,value,effect> using the matching operator <operator>.
+                      properties:
+                        effect:
+                          description: |-
+                            Effect indicates the taint effect to match. Empty means match all taint effects.
+                            When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                          type: string
+                        key:
+                          description: |-
+                            Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                            If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                          type: string
+                        operator:
+                          description: |-
+                            Operator represents a key's relationship to the value.
+                            Valid operators are Exists and Equal. Defaults to Equal.
+                            Exists is equivalent to wildcard for value, so that a pod can
+                            tolerate all taints of a particular category.
+                          type: string
+                        tolerationSeconds:
+                          description: |-
+                            TolerationSeconds represents the period of time the toleration (which must be
+                            of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                            it is not set, which means tolerate the taint forever (do not evict). Zero and
+                            negative values will be treated as 0 (evict immediately) by the system.
+                          format: int64
+                          type: integer
+                        value:
+                          description: |-
+                            Value is the taint value the toleration matches to.
+                            If the operator is Exists, the value should be empty, otherwise just a regular string.
+                          type: string
+                      type: object
+                    type: array
                   trigger:
                     description: Trigger defines the scenarios where a new build should
                       be triggered.

deploy/crds/shipwright.io_builds.yaml

@@ -2912,6 +2912,45 @@ spec:
                   should take to execute.
                 format: duration
                 type: string
+              tolerations:
+                description: If specified, the pod's tolerations.
+                items:
+                  description: |-
+                    The pod this Toleration is attached to tolerates any taint that matches
+                    the triple <key,value,effect> using the matching operator <operator>.
+                  properties:
+                    effect:
+                      description: |-
+                        Effect indicates the taint effect to match. Empty means match all taint effects.
+                        When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                      type: string
+                    key:
+                      description: |-
+                        Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                        If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                      type: string
+                    operator:
+                      description: |-
+                        Operator represents a key's relationship to the value.
+                        Valid operators are Exists and Equal. Defaults to Equal.
+                        Exists is equivalent to wildcard for value, so that a pod can
+                        tolerate all taints of a particular category.
+                      type: string
+                    tolerationSeconds:
+                      description: |-
+                        TolerationSeconds represents the period of time the toleration (which must be
+                        of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                        it is not set, which means tolerate the taint forever (do not evict). Zero and
+                        negative values will be treated as 0 (evict immediately) by the system.
+                      format: int64
+                      type: integer
+                    value:
+                      description: |-
+                        Value is the taint value the toleration matches to.
+                        If the operator is Exists, the value should be empty, otherwise just a regular string.
+                      type: string
+                  type: object
+                type: array
               trigger:
                 description: Trigger defines the scenarios where a new build should
                   be triggered.

pkg/apis/build/v1beta1/build_types.go

@@ -78,6 +78,8 @@ const (
 	OutputTimestampNotValid BuildReason = "OutputTimestampNotValid"
 	// NodeSelectorNotValid indicates that the nodeSelector value is not valid
 	NodeSelectorNotValid BuildReason = "NodeSelectorNotValid"
+	// TolerationNotValid indicates that the Toleration value is not valid
+	TolerationNotValid BuildReason = "TolerationNotValid"
 
 	// AllValidationsSucceeded indicates a Build was successfully validated
 	AllValidationsSucceeded = "all validations succeeded"
@@ -183,6 +185,12 @@ type BuildSpec struct {
 	//
 	// +optional
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+
+	// If specified, the pod's tolerations.
+	// +optional
+	// +patchMergeKey=Key
+	// +patchStrategy=merge
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty" patchStrategy:"merge" patchMergeKey:"Key"`
 }
 
 // BuildVolume is a volume that will be mounted in build pod during build step

pkg/apis/build/v1beta1/buildrun_types.go

@@ -115,6 +115,12 @@ type BuildRunSpec struct {
 	//
 	// +optional
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+
+	// If specified, the pod's tolerations.
+	// +optional
+	// +patchMergeKey=Key
+	// +patchStrategy=merge
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty" patchStrategy:"merge" patchMergeKey:"Key"`
 }
 
 // BuildRunRequestedState defines the buildrun state the user can provide to override whatever is the current state.

pkg/reconciler/build/build.go

@@ -34,6 +34,7 @@ var validationTypes = [...]string{
 	validate.Envs,
 	validate.Triggers,
 	validate.NodeSelector,
+	validate.Tolerations,
 }
 
 // ReconcileBuild reconciles a Build object

pkg/reconciler/buildrun/buildrun.go

@@ -161,6 +161,7 @@ func (r *ReconcileBuildRun) Reconcile(ctx context.Context, request reconcile.Req
 						validate.NewBuildName(build),
 						validate.NewEnv(build),
 						validate.NewNodeSelector(build),
+						validate.NewTolerations(build),
 					)
 
 					// an internal/technical error during validation happened

pkg/reconciler/buildrun/resources/taskrun.go

@@ -7,6 +7,7 @@ package resources
 import (
 	"fmt"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -235,12 +236,21 @@ func GenerateTaskRun(
 		},
 	}
 
+	taskRunPodTemplate := &pod.PodTemplate{}
 	// Merge Build and BuildRun NodeSelectors, giving preference to BuildRun NodeSelector
 	taskRunNodeSelector := mergeMaps(build.Spec.NodeSelector, buildRun.Spec.NodeSelector)
 	if len(taskRunNodeSelector) > 0 {
-		expectedTaskRun.Spec.PodTemplate = &pod.PodTemplate{
-			NodeSelector: taskRunNodeSelector,
-		}
+		taskRunPodTemplate.NodeSelector = taskRunNodeSelector
+	}
+
+	// Merge Build and BuildRun Tolerations, giving preference to BuildRun Tolerations values
+	taskRunTolerations := mergeTolerations(build.Spec.Tolerations, buildRun.Spec.Tolerations)
+	if len(taskRunTolerations) > 0 {
+		taskRunPodTemplate.Tolerations = taskRunTolerations
+	}
+
+	if !(taskRunPodTemplate.Equals(&pod.PodTemplate{})) {
+		expectedTaskRun.Spec.PodTemplate = taskRunPodTemplate
 	}
 
 	// assign the annotations from the build strategy, filter out those that should not be propagated
@@ -354,6 +364,21 @@ func effectiveTimeout(build *buildv1beta1.Build, buildRun *buildv1beta1.BuildRun
 	return nil
 }
 
+// mergeTolerations merges the values for Spec.Tolerations in the given Build and BuildRun objects, with values in the BuildRun object overriding values
+// in the Build object (if present).
+func mergeTolerations(buildTolerations []corev1.Toleration, buildRunTolerations []corev1.Toleration) []corev1.Toleration {
+	mergedTolerations := []corev1.Toleration{}
+	mergedTolerations = append(mergedTolerations, buildRunTolerations...)
+	for _, toleration := range buildTolerations {
+		if !slices.ContainsFunc(mergedTolerations, func(t corev1.Toleration) bool {
+			return t.Key == toleration.Key
+		}) {
+			mergedTolerations = append(mergedTolerations, toleration)
+		}
+	}
+	return mergedTolerations
+}
+
 // isPropagatableAnnotation filters the last-applied-configuration annotation from kubectl because this would break the meaning of this annotation on the target object;
 // also, annotations using our own custom resource domains are filtered out because we have no annotations with a semantic for both TaskRun and Pod
 func isPropagatableAnnotation(key string) bool {

pkg/validate/tolerations.go

@@ -0,0 +1,66 @@
+// Copyright The Shipwright Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package validate
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/utils/ptr"
+
+	build "github.com/shipwright-io/build/pkg/apis/build/v1beta1"
+)
+
+// TolerationsRef contains all required fields
+// to validate tolerations
+type TolerationsRef struct {
+	Build *build.Build // build instance for analysis
+}
+
+func NewTolerations(build *build.Build) *TolerationsRef {
+	return &TolerationsRef{build}
+}
+
+// ValidatePath implements BuildPath interface and validates
+// that tolerations key/operator/value are valid
+func (b *TolerationsRef) ValidatePath(_ context.Context) error {
+	for i, toleration := range b.Build.Spec.Tolerations {
+		// validate Key
+		if errs := validation.IsQualifiedName(toleration.Key); errs != nil {
+			b.Build.Status.Reason = ptr.To(build.TolerationNotValid)
+			b.Build.Status.Message = ptr.To(strings.Join(errs, ", "))
+		}
+		// validate Operator
+		if !((toleration.Operator == v1.TolerationOpExists) || (toleration.Operator == v1.TolerationOpEqual)) {
+			b.Build.Status.Reason = ptr.To(build.TolerationNotValid)
+			b.Build.Status.Message = ptr.To(fmt.Sprintf("Toleration operator not valid. Must be one of: '%v', '%v'", v1.TolerationOpExists, v1.TolerationOpEqual))
+		}
+		// validate Value
+		if errs := validation.IsValidLabelValue(toleration.Value); errs != nil {
+			b.Build.Status.Reason = ptr.To(build.TolerationNotValid)
+			b.Build.Status.Message = ptr.To(strings.Join(errs, ", "))
+		}
+		// validate Effect, of which only "NoSchedule" is supported
+		if toleration.Effect != v1.TaintEffectNoSchedule {
+			if !(len(toleration.Effect) > 0) {
+				// Effect was not specified, set it to the supported default
+				b.Build.Spec.Tolerations[i].Effect = v1.TaintEffectNoSchedule
+			} else {
+				b.Build.Status.Reason = ptr.To(build.TolerationNotValid)
+				b.Build.Status.Message = ptr.To(fmt.Sprintf("Only the '%v' toleration effect is supported.", v1.TaintEffectNoSchedule))
+			}
+		}
+		// validate TolerationSeconds, which should not be specified
+		if toleration.TolerationSeconds != nil {
+			b.Build.Status.Reason = ptr.To(build.TolerationNotValid)
+			b.Build.Status.Message = ptr.To("Specifying TolerationSeconds is not supported.")
+		}
+	}
+
+	return nil
+}