jitter initial probe execution time

Signed-off-by: Maskym Vavilov <[email protected]>

internal/common/helper.go

@@ -6,18 +6,22 @@ import (
 	"k8s.io/apimachinery/pkg/util/rand"
 )
 
-// RandomizeDuration randomizes duration for a given variance.
+// RandomizeValidationDuration randomizes duration for a given variance with a min value of 1 sec
 // variance is expected to be of a format 0.1 for 10%, 0.5 for 50% and so on
-func RandomizeDuration(variance float64, duration time.Duration) time.Duration {
+func RandomizeValidationDuration(variance float64, duration time.Duration) time.Duration {
 	// do not allow less than a second requeue
 	if duration.Milliseconds() < 1000 {
 		duration = time.Second * 1
 	}
 	// we won't go smaller than a second - using milliseconds to have a relatively big number to randomize
-	millisecond := float64(duration.Milliseconds())
+	return RandomizeDuration(variance, float64(duration.Milliseconds()))
+}
 
-	upperLimit := millisecond * (1.0 + variance)
-	lowerLimit := millisecond * (1.0 - variance)
+// RandomizeDuration randomizes duration for a given variance.
+// variance is expected to be of a format 0.1 for 10%, 0.5 for 50% and so on
+func RandomizeDuration(variance, duration float64) time.Duration {
+	upperLimit := duration * (1.0 + variance)
+	lowerLimit := duration * (1.0 - variance)
 
 	return time.Millisecond * time.Duration(rand.Int63nRange(
 		int64(lowerLimit),

internal/common/helper_test.go

@@ -25,8 +25,8 @@ func TestRandomizeDuration(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			i := 0
 			for i < testIterations {
-				if got := RandomizeDuration(tt.variance, tt.duration); !isValidVariance(tt.duration, got, tt.variance) {
-					t.Errorf("RandomizeDuration() invalid randomization; got = %v", got.String())
+				if got := RandomizeValidationDuration(tt.variance, tt.duration); !isValidVariance(tt.duration, got, tt.variance) {
+					t.Errorf("RandomizeValidationDuration() invalid randomization; got = %v", got.String())
 				}
 				i++
 			}

internal/controller/dnsrecord_controller.go

@@ -99,7 +99,7 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	defer postReconcile(ctx)
 
 	// randomize validation reconcile delay
-	randomizedValidationRequeue = common.RandomizeDuration(validationRequeueVariance, defaultValidationRequeue)
+	randomizedValidationRequeue = common.RandomizeValidationDuration(validationRequeueVariance, defaultValidationRequeue)
 
 	previous := &v1alpha1.DNSRecord{}
 	err := r.Client.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, previous)

internal/probes/worker.go

@@ -13,16 +13,21 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/kuadrant/dns-operator/api/v1alpha1"
+	"github.com/kuadrant/dns-operator/internal/common"
 	"github.com/kuadrant/dns-operator/internal/common/slice"
 	"github.com/kuadrant/dns-operator/internal/metrics"
 )
 
 var (
 	ExpectedResponses = []int{200, 201}
+
+	ProbeDelay = float64(time.Second.Milliseconds())
 )
 
 const (
 	PROBE_TIMEOUT = 3 * time.Second
+
+	ProbeDelayVariance = 0.5
 )
 
 type ProbeResult struct {
@@ -230,6 +235,9 @@ func (w *Probe) Start(clientctx context.Context, k8sClient client.Client, probe
 	logger.V(1).Info("health: starting new worker for probe")
 
 	go func() {
+		// jitter probe execution so we aren't starting at the same time
+		time.Sleep(common.RandomizeDuration(ProbeDelayVariance, ProbeDelay))
+
 		metrics.ProbeCounter.WithLabelValues(probe.Name, probe.Namespace, probe.Spec.Hostname).Inc()
 		//each time the probe executes it will send a result on the channel returned by ExecuteProbe until the probe is cancelled. The probe can be cancelled by a new spec being created for the healthcheck or on shutdown
 		for probeResult := range w.ExecuteProbe(ctx, probe) {