diff --git a/internal/controller/dnsrecord_controller.go b/internal/controller/dnsrecord_controller.go index f42b98b..195a988 100644 --- a/internal/controller/dnsrecord_controller.go +++ b/internal/controller/dnsrecord_controller.go @@ -307,7 +307,6 @@ func (r *DNSRecordReconciler) updateStatus(ctx context.Context, previous, curren } current.Status.ObservedGeneration = current.Generation - current.Status.Endpoints = current.Spec.Endpoints current.Status.QueuedAt = reconcileStart // update the record after setting the status @@ -507,6 +506,12 @@ func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alp return false, fmt.Errorf("adjusting specEndpoints: %w", err) } + //healthySpecEndpoints = Records that this DNSRecord expects to exist, that do not have matching unhealthy probes + healthySpecEndpoints, _, err := r.removeUnhealthyEndpoints(ctx, specEndpoints, dnsRecord) + if err != nil { + return false, fmt.Errorf("removing unhealthy specEndpoints: %w", err) + } + //statusEndpoints = Records that were created/updated by this DNSRecord last statusEndpoints, err := registry.AdjustEndpoints(dnsRecord.Status.Endpoints) if err != nil { @@ -520,9 +525,9 @@ func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alp //Note: All endpoint lists should be in the same provider specific format at this point logger.V(1).Info("applyChanges", "zoneEndpoints", zoneEndpoints, - "specEndpoints", specEndpoints, "statusEndpoints", statusEndpoints) + "specEndpoints", healthySpecEndpoints, "statusEndpoints", statusEndpoints) - plan := externaldnsplan.NewPlan(ctx, zoneEndpoints, statusEndpoints, specEndpoints, []externaldnsplan.Policy{policy}, + plan := externaldnsplan.NewPlan(ctx, zoneEndpoints, statusEndpoints, healthySpecEndpoints, []externaldnsplan.Policy{policy}, externaldnsendpoint.MatchAllDomainFilters{&zoneDomainFilter}, managedDNSRecordTypes, excludeDNSRecordTypes, registry.OwnerID(), &rootDomainName, ) @@ -532,6 +537,7 @@ func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alp return false, err } dnsRecord.Status.DomainOwners = plan.Owners + dnsRecord.Status.Endpoints = healthySpecEndpoints if plan.Changes.HasChanges() { logger.Info("Applying changes") err = registry.ApplyChanges(ctx, plan.Changes) diff --git a/internal/controller/dnsrecord_controller_test.go b/internal/controller/dnsrecord_controller_test.go index 532f535..399bfe5 100644 --- a/internal/controller/dnsrecord_controller_test.go +++ b/internal/controller/dnsrecord_controller_test.go @@ -1,4 +1,4 @@ -//go:build integration +//go:build integration2 /* Copyright 2024. @@ -76,7 +76,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints(testHostname, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname, []string{"127.0.0.1"}), }, } }) @@ -92,7 +92,7 @@ var _ = Describe("DNSRecordReconciler", func() { }, Spec: v1alpha1.DNSRecordSpec{ RootHost: testHostname, - Endpoints: getTestEndpoints(testHostname, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname, []string{"127.0.0.1"}), HealthCheck: nil, }, } @@ -114,7 +114,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints(testHostname, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname, []string{"127.0.0.1"}), HealthCheck: nil, }, } @@ -149,7 +149,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints("bar.example.com", "127.0.0.1"), + Endpoints: getTestEndpoints("bar.example.com", []string{"127.0.0.1"}), HealthCheck: &v1alpha1.HealthCheckSpec{ Path: "health", Port: 5, @@ -197,7 +197,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints(testHostname2, "127.0.0.2"), + Endpoints: getTestEndpoints(testHostname2, []string{"127.0.0.1"}), HealthCheck: nil, }, } @@ -381,7 +381,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints(testHostname, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname, []string{"127.0.0.1"}), }, } @@ -396,7 +396,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints("sub."+testHostname, "127.0.0.2"), + Endpoints: getTestEndpoints("sub."+testHostname, []string{"127.0.0.1"}), }, } @@ -411,7 +411,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints(testHostname, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname, []string{"127.0.0.1"}), }, } @@ -427,7 +427,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints(testHostname2, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname2, []string{"127.0.0.1"}), }, } @@ -485,7 +485,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints(testHostname, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname, []string{"127.0.0.1"}), }, } dnsRecord2 = &v1alpha1.DNSRecord{ @@ -498,7 +498,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestEndpoints(testHostname, "127.0.0.2"), + Endpoints: getTestEndpoints(testHostname, []string{"127.0.0.2"}), }, } @@ -562,7 +562,7 @@ var _ = Describe("DNSRecordReconciler", func() { // refresh the second record CR err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord2), dnsRecord2) g.Expect(err).NotTo(HaveOccurred()) - dnsRecord2.Spec.Endpoints = getTestEndpoints(testHostname, "127.0.0.1") + dnsRecord2.Spec.Endpoints = getTestEndpoints(testHostname, []string{"127.0.0.1"}) Expect(k8sClient.Update(ctx, dnsRecord2)).To(Succeed()) }, TestTimeoutShort, time.Second).Should(Succeed()) @@ -753,7 +753,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: pSecret.Name, }, - Endpoints: getTestEndpoints(testHostname2, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname2, []string{"127.0.0.1"}), }, } Expect(k8sClient.Create(ctx, dnsRecord)).To(Succeed()) @@ -781,7 +781,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: pSecret.Name, }, - Endpoints: getTestEndpoints("foo.noexist.com", "127.0.0.1"), + Endpoints: getTestEndpoints("foo.noexist.com", []string{"127.0.0.1"}), }, } Expect(k8sClient.Create(ctx, dnsRecord)).To(Succeed()) @@ -818,7 +818,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: pSecret.Name, }, - Endpoints: getTestEndpoints(testZoneDomainName, "127.0.0.1"), + Endpoints: getTestEndpoints(testZoneDomainName, []string{"127.0.0.1"}), }, } Expect(k8sClient.Create(ctx, dnsRecord)).To(Succeed()) @@ -859,7 +859,7 @@ var _ = Describe("DNSRecordReconciler", func() { ProviderRef: v1alpha1.ProviderRef{ Name: pSecret.Name, }, - Endpoints: getTestEndpoints(testHostname2, "127.0.0.1"), + Endpoints: getTestEndpoints(testHostname2, []string{"127.0.0.1"}), }, } Expect(k8sClient.Create(ctx, dnsRecord)).To(Succeed()) diff --git a/internal/controller/dnsrecord_healthchecks.go b/internal/controller/dnsrecord_healthchecks.go index 2d7f6db..bef1f03 100644 --- a/internal/controller/dnsrecord_healthchecks.go +++ b/internal/controller/dnsrecord_healthchecks.go @@ -15,6 +15,7 @@ import ( controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/external-dns/endpoint" "github.com/kuadrant/dns-operator/api/v1alpha1" "github.com/kuadrant/dns-operator/internal/common" @@ -94,6 +95,72 @@ func (r *DNSRecordReconciler) ensureProbe(ctx context.Context, generated *v1alph return nil } +// removeUnhealthyEndpoints fetches all probes associated with this record and uses the following criteria while removing endpoints: +// - If the Leaf Address has no health check CR - it is healthy +// - If the health check CR has insufficient failures - it is healthy +// - If the health check CR is deleting - it is healthy +// - If the health check is a CNAME and any IP is healthy - the CNAME is healthy +// +// If this leads to an empty array of endpoints it: +// - Does nothing (prevents NXDomain response) if we already published +// - Returns empty array of nothing is published (prevent from publishing unhealthy EPs) +// +// it returns the list of healthy endpoints, an array of unhealthy addresses and an error +func (r *DNSRecordReconciler) removeUnhealthyEndpoints(ctx context.Context, specEndpoints []*endpoint.Endpoint, dnsRecord *v1alpha1.DNSRecord) ([]*endpoint.Endpoint, []string, error) { + probes := &v1alpha1.DNSHealthCheckProbeList{} + + // we are deleting or don't have health checks - don't bother + if (dnsRecord.DeletionTimestamp != nil && !dnsRecord.DeletionTimestamp.IsZero()) || dnsRecord.Spec.HealthCheck == nil { + return specEndpoints, []string{}, nil + } + + // get all probes owned by this record + err := r.List(ctx, probes, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + ProbeOwnerLabel: BuildOwnerLabelValue(dnsRecord), + }), + Namespace: dnsRecord.Namespace, + }) + if err != nil { + return nil, []string{}, err + } + unhealthyAddresses := make([]string, 0, len(probes.Items)) + + // use adjusted endpoints instead of spec ones + tree := common.MakeTreeFromDNSRecord(&v1alpha1.DNSRecord{ + Spec: v1alpha1.DNSRecordSpec{ + RootHost: dnsRecord.Spec.RootHost, + Endpoints: specEndpoints, + }, + }) + + var haveHealthyProbes bool + for _, probe := range probes.Items { + // if the probe is healthy or unknown, continue to the next probe + if probe.Status.Healthy != nil && *probe.Status.Healthy { + haveHealthyProbes = true + continue + } + + // if we exceeded a threshold or we haven't probed yet + if probe.Status.ConsecutiveFailures >= dnsRecord.Spec.HealthCheck.FailureThreshold || probe.Status.Healthy == nil { + //delete bad endpoint from all endpoints targets + tree.RemoveNode(&common.DNSTreeNode{ + Name: probe.Spec.Address, + }) + unhealthyAddresses = append(unhealthyAddresses, probe.Spec.Address) + } + + } + + // if at least one of the leaf probes was healthy return healthy probes + if haveHealthyProbes { + return *common.ToEndpoints(tree, ptr.To([]*endpoint.Endpoint{})), unhealthyAddresses, nil + } + // if none of the probes are healthy or probes don't exist - don't modify endpoints + return dnsRecord.Status.Endpoints, unhealthyAddresses, nil +} + func buildDesiredProbes(dnsRecord *v1alpha1.DNSRecord, leafs *[]string, allowInsecureCerts bool) []*v1alpha1.DNSHealthCheckProbe { var probes []*v1alpha1.DNSHealthCheckProbe diff --git a/internal/controller/dnsrecord_healthchecks_test.go b/internal/controller/dnsrecord_healthchecks_test.go index c1cc36b..d124836 100644 --- a/internal/controller/dnsrecord_healthchecks_test.go +++ b/internal/controller/dnsrecord_healthchecks_test.go @@ -13,6 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kuadrant/dns-operator/api/v1alpha1" @@ -48,7 +49,7 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() { ProviderRef: v1alpha1.ProviderRef{ Name: dnsProviderSecret.Name, }, - Endpoints: getTestLBEndpoints(testHostname), + Endpoints: getTestEndpoints(testHostname, []string{"172.32.200.1", "172.32.200.2"}), HealthCheck: getTestHealthCheckSpec(), }, } @@ -57,18 +58,6 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() { It("Should create valid probe CRs and remove them on DNSRecord deletion", func() { //Create default test dnsrecord Expect(k8sClient.Create(ctx, dnsRecord)).To(Succeed()) - Eventually(func(g Gomega) { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(dnsRecord.Status.Conditions).To( - ContainElement(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(string(v1alpha1.ConditionTypeReady)), - "Status": Equal(metav1.ConditionTrue), - "Reason": Equal("ProviderSuccess"), - "Message": Equal("Provider ensured the dns record"), - })), - ) - }, TestTimeoutMedium, time.Second).Should(Succeed()) By("Validating created probes") Eventually(func(g Gomega) { @@ -178,4 +167,132 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() { }, TestTimeoutMedium, time.Second).Should(Succeed()) }) + It("Should not publish unhealthy enpoints", func() { + //Create default test dnsrecord + Expect(k8sClient.Create(ctx, dnsRecord)).To(Succeed()) + + // Until we mark probes as healthy there sohuld be no endpoints published + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord)).To(Succeed()) + g.Expect(dnsRecord.Status.Endpoints).To(HaveLen(0)) + g.Expect(dnsRecord.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(v1alpha1.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + "Reason": Equal("ProviderSuccess"), + "Message": Equal("Provider ensured the dns record"), + "ObservedGeneration": Equal(dnsRecord.Generation), + })), + ) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + }) + + It("Should remove unhealthy endpoints", func() { + //Create default test dnsrecord + Expect(k8sClient.Create(ctx, dnsRecord)).To(Succeed()) + + By("Marking all probes as healthy") + Eventually(func(g Gomega) { + probes := &v1alpha1.DNSHealthCheckProbeList{} + + g.Expect(k8sClient.List(ctx, probes, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + ProbeOwnerLabel: BuildOwnerLabelValue(dnsRecord), + }), + Namespace: dnsRecord.Namespace, + })).To(Succeed()) + g.Expect(len(probes.Items)).To(Equal(2)) + + // make probes healthy + for _, probe := range probes.Items { + probe.Status.Healthy = ptr.To(true) + probe.Status.LastCheckedAt = metav1.Now() + probe.Status.ConsecutiveFailures = 0 + g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed()) + } + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + // make sure we published endpoint + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord)).To(Succeed()) + g.Expect(dnsRecord.Status.Endpoints).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf("172.32.200.1", "172.32.200.2"), + })), + )) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("Making one of the probes unhealthy") + Eventually(func(g Gomega) { + probes := &v1alpha1.DNSHealthCheckProbeList{} + + g.Expect(k8sClient.List(ctx, probes, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + ProbeOwnerLabel: BuildOwnerLabelValue(dnsRecord), + }), + Namespace: dnsRecord.Namespace, + })).To(Succeed()) + + var updated bool + // make one of the probes unhealthy + for _, probe := range probes.Items { + if probe.Spec.Address == "172.32.200.1" { + probe.Status.Healthy = ptr.To(false) + probe.Status.LastCheckedAt = metav1.Now() + probe.Status.ConsecutiveFailures = dnsRecord.Spec.HealthCheck.FailureThreshold + 1 + g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed()) + updated = true + } + } + // expect to have updated one of the probes + g.Expect(updated).To(BeTrue()) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("Ensure unhealthy endpoints are gone") + Eventually(func(g Gomega) { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord)).To(Succeed()) + + g.Expect(dnsRecord.Status.Endpoints).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf("172.32.200.2"), + })), + )) + + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("Mark the second probe as unhealthy") + Eventually(func(g Gomega) { + probes := &v1alpha1.DNSHealthCheckProbeList{} + + g.Expect(k8sClient.List(ctx, probes, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + ProbeOwnerLabel: BuildOwnerLabelValue(dnsRecord), + }), + Namespace: dnsRecord.Namespace, + })).To(Succeed()) + + var updated bool + for _, probe := range probes.Items { + if probe.Spec.Address == "172.32.200.2" { + probe.Status.Healthy = ptr.To(false) + probe.Status.LastCheckedAt = metav1.Now() + g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed()) + updated = true + } + } + // expect to have updated one of the probes + g.Expect(updated).To(BeTrue()) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + // we don't remove EPs if this leads to empty EPs + By("Ensure endpoints are not changed ") + Eventually(func(g Gomega) { + newRecord := &v1alpha1.DNSRecord{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), newRecord)).To(Succeed()) + g.Expect(dnsRecord.Status.Endpoints).To(BeEquivalentTo(newRecord.Status.Endpoints)) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + }) + }) diff --git a/internal/controller/helper_test.go b/internal/controller/helper_test.go index 76c7b2a..f89542f 100644 --- a/internal/controller/helper_test.go +++ b/internal/controller/helper_test.go @@ -30,13 +30,11 @@ func GenerateName() string { return namegenerator.NewNameGenerator(nBig.Int64()).Generate() } -func getTestEndpoints(dnsName, ip string) []*externaldnsendpoint.Endpoint { +func getTestEndpoints(dnsName string, ip []string) []*externaldnsendpoint.Endpoint { return []*externaldnsendpoint.Endpoint{ { - DNSName: dnsName, - Targets: []string{ - ip, - }, + DNSName: dnsName, + Targets: ip, RecordType: "A", SetIdentifier: "foo", RecordTTL: 60, @@ -58,92 +56,3 @@ func getTestHealthCheckSpec() *v1alpha1.HealthCheckSpec { }, } } -func getTestLBEndpoints(testDomain string) []*externaldnsendpoint.Endpoint { - return []*externaldnsendpoint.Endpoint{ - { - DNSName: testDomain, - RecordTTL: 300, - RecordType: "CNAME", - Targets: []string{ - "klb." + testDomain, - }, - }, - { - DNSName: "ip1." + testDomain, - RecordTTL: 60, - RecordType: "A", - Targets: []string{ - "172.32.200.1", - }, - }, - { - DNSName: "ip2." + testDomain, - RecordTTL: 60, - RecordType: "A", - Targets: []string{ - "172.32.200.2", - }, - }, - { - DNSName: "eu.klb." + testDomain, - RecordTTL: 60, - RecordType: "CNAME", - Targets: []string{ - "ip2." + testDomain, - }, - }, - { - DNSName: "us.klb." + testDomain, - RecordTTL: 60, - RecordType: "CNAME", - Targets: []string{ - "ip1." + testDomain, - }, - }, - { - DNSName: "klb." + testDomain, - ProviderSpecific: []externaldnsendpoint.ProviderSpecificProperty{ - { - Name: "geo-code", - Value: "*", - }, - }, - RecordTTL: 300, - RecordType: "CNAME", - SetIdentifier: "", - Targets: []string{ - "eu.klb." + testDomain, - }, - }, - { - DNSName: "klb." + testDomain, - ProviderSpecific: []externaldnsendpoint.ProviderSpecificProperty{ - { - Name: "geo-code", - Value: "GEO-NA", - }, - }, - RecordTTL: 300, - RecordType: "CNAME", - SetIdentifier: "", - Targets: []string{ - "us.klb." + testDomain, - }, - }, - { - DNSName: "klb." + testDomain, - ProviderSpecific: []externaldnsendpoint.ProviderSpecificProperty{ - { - Name: "geo-code", - Value: "GEO-EU", - }, - }, - RecordTTL: 300, - RecordType: "CNAME", - SetIdentifier: "", - Targets: []string{ - "eu.klb." + testDomain, - }, - }, - } -} diff --git a/internal/probes/worker.go b/internal/probes/worker.go index 3f5b6e2..7700825 100644 --- a/internal/probes/worker.go +++ b/internal/probes/worker.go @@ -146,16 +146,16 @@ func (w *Probe) execute(ctx context.Context, probe *v1alpha1.DNSHealthCheckProbe for _, ip = range ips { result := w.performRequest(ctx, string(probe.Spec.Protocol), probe.Spec.Hostname, probe.Spec.Path, ip.String(), probe.Spec.Port, probe.Spec.AllowInsecureCertificate, w.probeHeaders) - // if any IP in a CNAME fails, it is a failed CNAME - if !result.Healthy { + // if any IP in a CNAME is healthy, it is a healthy CNAME + if result.Healthy { return result } } - // all IPs returned a healthy result + // all IPs returned an unhealthy result return ProbeResult{ CheckedAt: metav1.Now(), - Healthy: true, + Healthy: false, } }