diff --git a/internal/common/helper.go b/internal/common/helper.go index 3a72c9a..bc343e1 100644 --- a/internal/common/helper.go +++ b/internal/common/helper.go @@ -6,18 +6,22 @@ import ( "k8s.io/apimachinery/pkg/util/rand" ) -// RandomizeDuration randomizes duration for a given variance. +// RandomizeValidationDuration randomizes duration for a given variance with a min value of 1 sec // variance is expected to be of a format 0.1 for 10%, 0.5 for 50% and so on -func RandomizeDuration(variance float64, duration time.Duration) time.Duration { +func RandomizeValidationDuration(variance float64, duration time.Duration) time.Duration { // do not allow less than a second requeue if duration.Milliseconds() < 1000 { duration = time.Second * 1 } // we won't go smaller than a second - using milliseconds to have a relatively big number to randomize - millisecond := float64(duration.Milliseconds()) + return RandomizeDuration(variance, float64(duration.Milliseconds())) +} - upperLimit := millisecond * (1.0 + variance) - lowerLimit := millisecond * (1.0 - variance) +// RandomizeDuration randomizes duration for a given variance. +// variance is expected to be of a format 0.1 for 10%, 0.5 for 50% and so on +func RandomizeDuration(variance, duration float64) time.Duration { + upperLimit := duration * (1.0 + variance) + lowerLimit := duration * (1.0 - variance) return time.Millisecond * time.Duration(rand.Int63nRange( int64(lowerLimit), diff --git a/internal/common/helper_test.go b/internal/common/helper_test.go index 50a9961..0ae62d4 100644 --- a/internal/common/helper_test.go +++ b/internal/common/helper_test.go @@ -25,8 +25,8 @@ func TestRandomizeDuration(t *testing.T) { t.Run(tt.name, func(t *testing.T) { i := 0 for i < testIterations { - if got := RandomizeDuration(tt.variance, tt.duration); !isValidVariance(tt.duration, got, tt.variance) { - t.Errorf("RandomizeDuration() invalid randomization; got = %v", got.String()) + if got := RandomizeValidationDuration(tt.variance, tt.duration); !isValidVariance(tt.duration, got, tt.variance) { + t.Errorf("RandomizeValidationDuration() invalid randomization; got = %v", got.String()) } i++ } diff --git a/internal/controller/dnsrecord_controller.go b/internal/controller/dnsrecord_controller.go index a10a83b..b28e7f7 100644 --- a/internal/controller/dnsrecord_controller.go +++ b/internal/controller/dnsrecord_controller.go @@ -99,7 +99,7 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( defer postReconcile(ctx) // randomize validation reconcile delay - randomizedValidationRequeue = common.RandomizeDuration(validationRequeueVariance, defaultValidationRequeue) + randomizedValidationRequeue = common.RandomizeValidationDuration(validationRequeueVariance, defaultValidationRequeue) previous := &v1alpha1.DNSRecord{} err := r.Client.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, previous) diff --git a/internal/probes/worker.go b/internal/probes/worker.go index 474de8f..7ec3874 100644 --- a/internal/probes/worker.go +++ b/internal/probes/worker.go @@ -13,16 +13,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "github.com/kuadrant/dns-operator/api/v1alpha1" + "github.com/kuadrant/dns-operator/internal/common" "github.com/kuadrant/dns-operator/internal/common/slice" "github.com/kuadrant/dns-operator/internal/metrics" ) var ( ExpectedResponses = []int{200, 201} + + ProbeDelay = float64(time.Second.Milliseconds()) ) const ( PROBE_TIMEOUT = 3 * time.Second + + ProbeDelayVariance = 0.5 ) type ProbeResult struct { @@ -230,6 +235,9 @@ func (w *Probe) Start(clientctx context.Context, k8sClient client.Client, probe logger.V(1).Info("health: starting new worker for probe") go func() { + // jitter probe execution so we aren't starting at the same time + time.Sleep(common.RandomizeDuration(ProbeDelayVariance, ProbeDelay)) + metrics.ProbeCounter.WithLabelValues(probe.Name, probe.Namespace, probe.Spec.Hostname).Inc() //each time the probe executes it will send a result on the channel returned by ExecuteProbe until the probe is cancelled. The probe can be cancelled by a new spec being created for the healthcheck or on shutdown for probeResult := range w.ExecuteProbe(ctx, probe) {