From 66668bd591dc1c085af58bd6aa1f17edf549c43a Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Wed, 18 Oct 2023 12:58:46 +0100 Subject: [PATCH] Add DNSPolicy Strategy Adds a strategy field to the DNSPolicy spec that determines how the policy with generate endpoints for any created DNSRecords. Two strategies are allowed, `simple` and `loadbalanced`. Simple will creates a single DNS record (A or CNAME) for each listener/hostname with all ip/hostnames as targets. LoadBalanced works as before by creating a more complex record structure with CNAMES and A records using Geo and Weighted routing strategies to achieve loadbalancing functionality. The strategy field is currently marked as immutable and it should not be chnaged after initial DNSPolicy creation. --- config/crd/bases/kuadrant.io_dnspolicies.yaml | 9 ++ pkg/apis/v1alpha1/dnspolicy_types.go | 12 ++ pkg/controllers/dnspolicy/dns_helper.go | 118 +++++++++++++----- .../dnspolicy/dnspolicy_dnsrecords.go | 2 +- 4 files changed, 107 insertions(+), 34 deletions(-) diff --git a/config/crd/bases/kuadrant.io_dnspolicies.yaml b/config/crd/bases/kuadrant.io_dnspolicies.yaml index 2c0aff3f9..98e617b8f 100644 --- a/config/crd/bases/kuadrant.io_dnspolicies.yaml +++ b/config/crd/bases/kuadrant.io_dnspolicies.yaml @@ -155,6 +155,14 @@ spec: type: integer type: object type: object + strategy: + enum: + - simple + - loadbalanced + type: string + x-kubernetes-validations: + - message: DNSPolicyStrategy is immutable + rule: self == oldSelf targetRef: description: PolicyTargetReference identifies an API object to apply policy to. This should be used as part of Policy resources that @@ -193,6 +201,7 @@ spec: - name type: object required: + - strategy - targetRef type: object status: diff --git a/pkg/apis/v1alpha1/dnspolicy_types.go b/pkg/apis/v1alpha1/dnspolicy_types.go index 5beb6c199..96e016d9b 100644 --- a/pkg/apis/v1alpha1/dnspolicy_types.go +++ b/pkg/apis/v1alpha1/dnspolicy_types.go @@ -25,6 +25,13 @@ import ( gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) +type DNSPolicyStrategy string + +const ( + SimpleStrategy DNSPolicyStrategy = "simple" + LoadBalancedStrategy DNSPolicyStrategy = "loadbalanced" +) + // DNSPolicySpec defines the desired state of DNSPolicy type DNSPolicySpec struct { @@ -37,6 +44,11 @@ type DNSPolicySpec struct { // +optional LoadBalancing *LoadBalancingSpec `json:"loadBalancing"` + + // +required + // +kubebuilder:validation:Enum=simple;loadbalanced + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="DNSPolicyStrategy is immutable" + Strategy DNSPolicyStrategy `json:"strategy"` } type LoadBalancingSpec struct { diff --git a/pkg/controllers/dnspolicy/dns_helper.go b/pkg/controllers/dnspolicy/dns_helper.go index 68c4d38ff..53fc4ff9c 100644 --- a/pkg/controllers/dnspolicy/dns_helper.go +++ b/pkg/controllers/dnspolicy/dns_helper.go @@ -33,6 +33,7 @@ const ( ) var ( + ErrUnknownDNSStrategy = fmt.Errorf("unknown dns policy strategy") ErrNoManagedZoneForHost = fmt.Errorf("no managed zone for host") ErrAlreadyAssigned = fmt.Errorf("managed host already assigned") ) @@ -147,7 +148,75 @@ func withGatewayListener[T metav1.Object](gateway common.GatewayWrapper, listene return obj } -// setEndpoints sets the endpoints for the given MultiClusterGatewayTarget +func (dh *dnsHelper) setEndpoints(ctx context.Context, mcgTarget *dns.MultiClusterGatewayTarget, dnsRecord *v1alpha1.DNSRecord, listener gatewayv1beta1.Listener, strategy v1alpha1.DNSPolicyStrategy) error { + old := dnsRecord.DeepCopy() + gwListenerHost := string(*listener.Hostname) + var endpoints []*v1alpha1.Endpoint + + //Health Checks currently modify endpoints so we have to keep existing ones in order to not lose health check ids + currentEndpoints := make(map[string]*v1alpha1.Endpoint, len(dnsRecord.Spec.Endpoints)) + for _, endpoint := range dnsRecord.Spec.Endpoints { + currentEndpoints[endpoint.SetID()] = endpoint + } + + switch strategy { + case v1alpha1.SimpleStrategy: + endpoints = dh.getSimpleEndpoints(mcgTarget, gwListenerHost, currentEndpoints) + case v1alpha1.LoadBalancedStrategy: + endpoints = dh.getLoadBalancedEndpoints(mcgTarget, gwListenerHost, currentEndpoints) + default: + return fmt.Errorf("%w : %s", ErrUnknownDNSStrategy, strategy) + } + + sort.Slice(endpoints, func(i, j int) bool { + return endpoints[i].SetID() < endpoints[j].SetID() + }) + + dnsRecord.Spec.Endpoints = endpoints + + if !equality.Semantic.DeepEqual(old, dnsRecord) { + return dh.Update(ctx, dnsRecord) + } + + return nil +} + +// getSimpleEndpoints sets the endpoints for the given MultiClusterGatewayTarget using the simple DNS policy strategy + +func (dh *dnsHelper) getSimpleEndpoints(mcgTarget *dns.MultiClusterGatewayTarget, hostname string, currentEndpoints map[string]*v1alpha1.Endpoint) []*v1alpha1.Endpoint { + + var ( + endpoints []*v1alpha1.Endpoint + ipValues []string + hostValues []string + endpoint *v1alpha1.Endpoint + ) + + for _, cgwTarget := range mcgTarget.ClusterGatewayTargets { + for _, gwa := range cgwTarget.GatewayAddresses { + if *gwa.Type == gatewayv1beta1.IPAddressType { + ipValues = append(ipValues, gwa.Value) + } else { + hostValues = append(hostValues, gwa.Value) + } + } + } + + if len(ipValues) > 0 { + endpoint = createOrUpdateEndpoint(hostname, ipValues, v1alpha1.ARecordType, "", dns.DefaultTTL, currentEndpoints) + endpoints = append(endpoints, endpoint) + } + + //ToDO This is what external-dns does, but not sure it will actually work since you can't have CNAME records with multiple values afaik + if len(hostValues) > 0 { + endpoint = createOrUpdateEndpoint(hostname, hostValues, v1alpha1.CNAMERecordType, "", dns.DefaultTTL, currentEndpoints) + endpoints = append(endpoints, endpoint) + } + + return endpoints +} + +// getLoadBalancedEndpoints sets the endpoints for the given MultiClusterGatewayTarget using the loadbalanced DNS policy strategy // // Builds an array of v1alpha1.Endpoint resources and sets them on the given DNSRecord. The endpoints expected are calculated // from the MultiClusterGatewayTarget using the target Gateway (MultiClusterGatewayTarget.Gateway), the LoadBalancing Spec @@ -186,23 +255,15 @@ func withGatewayListener[T metav1.Object](gateway common.GatewayWrapper, listene // ab2.lb-a1b2.shop.example.com A 192.22.2.3 // ab3.lb-a1b2.shop.example.com A 192.22.2.4 -func (dh *dnsHelper) setEndpoints(ctx context.Context, mcgTarget *dns.MultiClusterGatewayTarget, dnsRecord *v1alpha1.DNSRecord, listener gatewayv1beta1.Listener) error { +func (dh *dnsHelper) getLoadBalancedEndpoints(mcgTarget *dns.MultiClusterGatewayTarget, hostname string, currentEndpoints map[string]*v1alpha1.Endpoint) []*v1alpha1.Endpoint { - old := dnsRecord.DeepCopy() - gwListenerHost := string(*listener.Hostname) - cnameHost := gwListenerHost - if isWildCardListener(listener) { - cnameHost = strings.Replace(gwListenerHost, "*.", "", -1) - } - - //Health Checks currently modify endpoints so we have to keep existing ones in order to not lose health check ids - currentEndpoints := make(map[string]*v1alpha1.Endpoint, len(dnsRecord.Spec.Endpoints)) - for _, endpoint := range dnsRecord.Spec.Endpoints { - currentEndpoints[endpoint.SetID()] = endpoint + cnameHost := hostname + if isWildCardHost(hostname) { + cnameHost = strings.Replace(hostname, "*.", "", -1) } var ( - newEndpoints []*v1alpha1.Endpoint + endpoints []*v1alpha1.Endpoint endpoint *v1alpha1.Endpoint defaultEndpoint *v1alpha1.Endpoint ) @@ -239,7 +300,7 @@ func (dh *dnsHelper) setEndpoints(ctx context.Context, mcgTarget *dns.MultiClust if len(clusterEndpoints) == 0 { continue } - newEndpoints = append(newEndpoints, clusterEndpoints...) + endpoints = append(endpoints, clusterEndpoints...) //Create lbName CNAME (lb-a1b2.shop.example.com -> default.lb-a1b2.shop.example.com) endpoint = createOrUpdateEndpoint(lbName, []string{geoLbName}, v1alpha1.CNAMERecordType, string(geoCode), dns.DefaultCnameTTL, currentEndpoints) @@ -256,28 +317,19 @@ func (dh *dnsHelper) setEndpoints(ctx context.Context, mcgTarget *dns.MultiClust endpoint.SetProviderSpecific(dns.ProviderSpecificGeoCode, string(geoCode)) - newEndpoints = append(newEndpoints, endpoint) + endpoints = append(endpoints, endpoint) } - if len(newEndpoints) > 0 { - // Add the `defaultEndpoint`, this should always be set by this point if `newEndpoints` isn't empty + if len(endpoints) > 0 { + // Add the `defaultEndpoint`, this should always be set by this point if `endpoints` isn't empty defaultEndpoint.SetProviderSpecific(dns.ProviderSpecificGeoCode, string(dns.WildcardGeo)) - newEndpoints = append(newEndpoints, defaultEndpoint) + endpoints = append(endpoints, defaultEndpoint) //Create gwListenerHost CNAME (shop.example.com -> lb-a1b2.shop.example.com) - endpoint = createOrUpdateEndpoint(gwListenerHost, []string{lbName}, v1alpha1.CNAMERecordType, "", dns.DefaultCnameTTL, currentEndpoints) - newEndpoints = append(newEndpoints, endpoint) + endpoint = createOrUpdateEndpoint(hostname, []string{lbName}, v1alpha1.CNAMERecordType, "", dns.DefaultCnameTTL, currentEndpoints) + endpoints = append(endpoints, endpoint) } - sort.Slice(newEndpoints, func(i, j int) bool { - return newEndpoints[i].SetID() < newEndpoints[j].SetID() - }) - - dnsRecord.Spec.Endpoints = newEndpoints - - if !equality.Semantic.DeepEqual(old, dnsRecord) { - return dh.Update(ctx, dnsRecord) - } - return nil + return endpoints } func createOrUpdateEndpoint(dnsName string, targets v1alpha1.Targets, recordType v1alpha1.DNSRecordType, setIdentifier string, @@ -374,8 +426,8 @@ func (dh *dnsHelper) deleteDNSRecordForListener(ctx context.Context, owner metav return dh.Delete(ctx, &dnsRecord, &client.DeleteOptions{}) } -func isWildCardListener(l gatewayv1beta1.Listener) bool { - return strings.HasPrefix(string(*l.Hostname), "*") +func isWildCardHost(host string) bool { + return strings.HasPrefix(host, "*") } func (dh *dnsHelper) getDNSHealthCheckProbes(ctx context.Context, gateway *gatewayv1beta1.Gateway, dnsPolicy *v1alpha1.DNSPolicy) ([]*v1alpha1.DNSHealthCheckProbe, error) { diff --git a/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go b/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go index af5affa70..81e5dc41e 100644 --- a/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go +++ b/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go @@ -118,7 +118,7 @@ func (r *DNSPolicyReconciler) reconcileGatewayDNSRecords(ctx context.Context, ga return err } mcgTarget.RemoveUnhealthyGatewayAddresses(probes, listener) - if err := r.dnsHelper.setEndpoints(ctx, mcgTarget, dnsRecord, listener); err != nil { + if err := r.dnsHelper.setEndpoints(ctx, mcgTarget, dnsRecord, listener, dnsPolicy.Spec.Strategy); err != nil { return fmt.Errorf("failed to add dns record dnsTargets %s %v", err, mcgTarget) } }