From b1fdcaa1e4f7600d4d404e5ba894b256aeb52ea6 Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Fri, 15 Mar 2024 11:24:40 +0000 Subject: [PATCH 1/3] external-dns: Replace v0.14.0 version with v0.14.0_kuadrant We will be using a forked version of external-dns for the time being and will track the latest release version (v0.14.0) + kuadrant related changes in https://github.com/Kuadrant/external-dns/tree/v0.14.0_kuadrant --- go.mod | 9 +++++++++ go.sum | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 8c0759a0..e29a4146 100644 --- a/go.mod +++ b/go.mod @@ -103,3 +103,12 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace sigs.k8s.io/external-dns => github.com/kuadrant/external-dns v0.0.0-20240315162317-073094ed9bea + +// To Update with changes from v0.14.0_kuadrant run: +// go mod edit --replace sigs.k8s.io/external-dns=github.com/kuadrant/external-dns@v0.14.0_kuadrant +// go mod tidy + +// To Update for local development +// go mod edit --replace sigs.k8s.io/external-dns=/home/mnairn/go/src/github.com/kubernetes-sigs/external-dns diff --git a/go.sum b/go.sum index b8b48dc6..df1aa9d1 100644 --- a/go.sum +++ b/go.sum @@ -482,6 +482,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kuadrant/external-dns v0.0.0-20240315162317-073094ed9bea h1:Ob3/nd2gCnlM1aa6YyKezQlmcUnBKT6zsS4l7FP7j6E= +github.com/kuadrant/external-dns v0.0.0-20240315162317-073094ed9bea/go.mod h1:d4Knr/BFz8U1Lc6yLhCzTRP6nJOz6fqR/MnqqJPcIlU= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -1169,8 +1171,6 @@ sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gE sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/controller-tools v0.3.1-0.20200517180335-820a4a27ea84/go.mod h1:enhtKGfxZD1GFEoMgP8Fdbu+uKQ/cq1/WGJhdVChfvI= -sigs.k8s.io/external-dns v0.14.0 h1:pgY3DdyoBei+ej1nyZUzRt9ECm9RRwb9s6/CPWe51tc= -sigs.k8s.io/external-dns v0.14.0/go.mod h1:d4Knr/BFz8U1Lc6yLhCzTRP6nJOz6fqR/MnqqJPcIlU= sigs.k8s.io/gateway-api v0.7.1 h1:Tts2jeepVkPA5rVG/iO+S43s9n7Vp7jCDhZDQYtPigQ= sigs.k8s.io/gateway-api v0.7.1/go.mod h1:Xv0+ZMxX0lu1nSSDIIPEfbVztgNZ+3cfiYrJsa2Ooso= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From cd1414cdfa7f3f00b0e59745ec1362a3fffd0e45 Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Mon, 11 Mar 2024 16:09:07 +0000 Subject: [PATCH 2/3] registry: Use txt registry for owned records Use the TXT Registry when an `ownerID` is specified for a record being processed. Adds an optional field `ownerID` to the DNSRecord spec. This is immutable since changing it after initial creation will likely not result in desirable behaviour. Update e2e tests to verify contents of zone after record creation. --- api/v1alpha1/dnsrecord_types.go | 20 +- api/v1alpha1/zz_generated.deepcopy.go | 5 + internal/controller/dnsrecord_controller.go | 12 +- .../controller/dnsrecord_controller_test.go | 148 ++++++ internal/controller/helper_test.go | 48 +- internal/controller/suite_test.go | 24 + test/e2e/single_cluster_record_test.go | 479 +++++++++++++++++- test/e2e/suite_test.go | 48 ++ 8 files changed, 746 insertions(+), 38 deletions(-) create mode 100644 internal/controller/dnsrecord_controller_test.go diff --git a/api/v1alpha1/dnsrecord_types.go b/api/v1alpha1/dnsrecord_types.go index 27f917c8..63228984 100644 --- a/api/v1alpha1/dnsrecord_types.go +++ b/api/v1alpha1/dnsrecord_types.go @@ -19,14 +19,16 @@ package v1alpha1 import ( "fmt" "strings" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" externaldns "sigs.k8s.io/external-dns/endpoint" + externaldnsprovider "sigs.k8s.io/external-dns/provider" + externaldnsregistry "sigs.k8s.io/external-dns/registry" ) // DNSRecordSpec defines the desired state of DNSRecord type DNSRecordSpec struct { - // OwnerID is a unique string used to identify all endpoints created by this kuadrant // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="OwnerID is immutable" // +optional @@ -108,6 +110,13 @@ const ( NSRecordType DNSRecordType = "NS" DefaultGeo string = "default" + + txtRegistryPrefix = "kuadrant-" + txtRegistrySuffix = "" + txtRegistryWildcardReplacement = "wildcard" + txtRegistryEncryptEnabled = false + txtRegistryEncryptAESKey = "" + txtRegistryCacheInterval = time.Duration(0) ) const WildcardPrefix = "*." @@ -143,6 +152,15 @@ func (s *DNSRecord) Validate() error { return nil } +func (s *DNSRecord) GetRegistry(provider externaldnsprovider.Provider, managedDNSRecordTypes, excludeDNSRecordTypes []string) (externaldnsregistry.Registry, error) { + if s.Spec.OwnerID != nil { + return externaldnsregistry.NewTXTRegistry(provider, txtRegistryPrefix, txtRegistrySuffix, *s.Spec.OwnerID, txtRegistryCacheInterval, + txtRegistryWildcardReplacement, managedDNSRecordTypes, excludeDNSRecordTypes, txtRegistryEncryptEnabled, []byte(txtRegistryEncryptAESKey)) + } else { + return externaldnsregistry.NewNoopRegistry(provider) + } +} + func init() { SchemeBuilder.Register(&DNSRecord{}, &DNSRecordList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 82cba831..a9642445 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -89,6 +89,11 @@ func (in *DNSRecordList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSRecordSpec) DeepCopyInto(out *DNSRecordSpec) { *out = *in + if in.OwnerID != nil { + in, out := &in.OwnerID, &out.OwnerID + *out = new(string) + **out = **in + } if in.RootHost != nil { in, out := &in.RootHost, &out.RootHost *out = new(string) diff --git a/internal/controller/dnsrecord_controller.go b/internal/controller/dnsrecord_controller.go index 89359e31..edc00af6 100644 --- a/internal/controller/dnsrecord_controller.go +++ b/internal/controller/dnsrecord_controller.go @@ -35,7 +35,6 @@ import ( externaldnsendpoint "sigs.k8s.io/external-dns/endpoint" externaldnsplan "sigs.k8s.io/external-dns/plan" externaldnsprovider "sigs.k8s.io/external-dns/provider" - externaldnsregistry "sigs.k8s.io/external-dns/registry" "github.com/kuadrant/dns-operator/api/v1alpha1" "github.com/kuadrant/dns-operator/internal/common/conditions" @@ -81,6 +80,9 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( logger.Info("Removing Finalizer", "name", DNSRecordFinalizer) controllerutil.RemoveFinalizer(dnsRecord, DNSRecordFinalizer) if err = r.Update(ctx, dnsRecord); client.IgnoreNotFound(err) != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } return ctrl.Result{}, err } return ctrl.Result{}, nil @@ -245,7 +247,10 @@ func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alp return err } - registry, err := externaldnsregistry.NewNoopRegistry(dnsProvider) + managedDNSRecordTypes := []string{externaldnsendpoint.RecordTypeA, externaldnsendpoint.RecordTypeAAAA, externaldnsendpoint.RecordTypeCNAME} + excludeDNSRecordTypes := []string{} + + registry, err := dnsRecord.GetRegistry(dnsProvider, managedDNSRecordTypes, excludeDNSRecordTypes) if err != nil { return err } @@ -256,9 +261,6 @@ func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alp return fmt.Errorf("unknown policy: %s", policyID) } - managedDNSRecordTypes := []string{externaldnsendpoint.RecordTypeA, externaldnsendpoint.RecordTypeAAAA, externaldnsendpoint.RecordTypeCNAME} - excludeDNSRecordTypes := []string{} - //If we are deleting set the expected endpoints to an empty array if isDelete { dnsRecord.Spec.Endpoints = []*externaldnsendpoint.Endpoint{} diff --git a/internal/controller/dnsrecord_controller_test.go b/internal/controller/dnsrecord_controller_test.go new file mode 100644 index 00000000..df6c0e30 --- /dev/null +++ b/internal/controller/dnsrecord_controller_test.go @@ -0,0 +1,148 @@ +//go:build integration + +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + externaldnsendpoint "sigs.k8s.io/external-dns/endpoint" + + "github.com/kuadrant/dns-operator/api/v1alpha1" + "github.com/kuadrant/dns-operator/internal/common/conditions" +) + +var _ = Describe("DNSRecordReconciler", func() { + var dnsRecord *v1alpha1.DNSRecord + var managedZone *v1alpha1.ManagedZone + var testNamespace string + + BeforeEach(func() { + CreateNamespace(&testNamespace) + + managedZone = testBuildManagedZone("mz-example-com", testNamespace, "example.com") + Expect(k8sClient.Create(ctx, managedZone)).To(Succeed()) + + dnsRecord = &v1alpha1.DNSRecord{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo.example.com", + Namespace: testNamespace, + }, + Spec: v1alpha1.DNSRecordSpec{ + ManagedZoneRef: &v1alpha1.ManagedZoneReference{ + Name: managedZone.Name, + }, + Endpoints: []*externaldnsendpoint.Endpoint{ + { + DNSName: "foo.example.com", + Targets: []string{ + "127.0.0.1", + }, + RecordType: "A", + SetIdentifier: "", + RecordTTL: 60, + Labels: nil, + ProviderSpecific: nil, + }, + }, + }, + } + }) + + AfterEach(func() { + if dnsRecord != nil { + err := k8sClient.Delete(ctx, dnsRecord) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + } + if managedZone != nil { + err := k8sClient.Delete(ctx, managedZone) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + } + }) + + It("should have ready condition with status true", func() { + Expect(k8sClient.Create(ctx, dnsRecord)).To(Succeed()) + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(dnsRecord.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(conditions.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + "Reason": Equal("ProviderSuccess"), + "Message": Equal("Provider ensured the dns record"), + "ObservedGeneration": Equal(dnsRecord.Generation), + })), + ) + g.Expect(dnsRecord.Finalizers).To(ContainElement(DNSRecordFinalizer)) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + }) + + It("should not allow ownerID to be updated once set", func() { + Expect(k8sClient.Create(ctx, dnsRecord)).To(BeNil()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(dnsRecord.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(conditions.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + "Reason": Equal("ProviderSuccess"), + "Message": Equal("Provider ensured the dns record"), + "ObservedGeneration": Equal(dnsRecord.Generation), + })), + ) + g.Expect(dnsRecord.Finalizers).To(ContainElement(DNSRecordFinalizer)) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + //Allows updating from not owned to owned + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + + dnsRecord.Spec.OwnerID = ptr.To("foobar") + err = k8sClient.Update(ctx, dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + //Does not allow ownerID to change once set + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(dnsRecord.Spec.OwnerID).To(PointTo(Equal("foobar"))) + + dnsRecord.Spec.OwnerID = ptr.To("foobarbaz") + err = k8sClient.Update(ctx, dnsRecord) + g.Expect(err).To(MatchError(ContainSubstring("OwnerID is immutable"))) + + err = k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(dnsRecord.Spec.OwnerID).To(PointTo(Equal("foobar"))) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + }) + +}) diff --git a/internal/controller/helper_test.go b/internal/controller/helper_test.go index c9a2ca86..4a7188c8 100644 --- a/internal/controller/helper_test.go +++ b/internal/controller/helper_test.go @@ -3,10 +3,11 @@ package controller import ( - "context" - "fmt" - "net/http" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kuadrantdnsv1alpha1 "github.com/kuadrant/dns-operator/api/v1alpha1" ) const ( @@ -17,32 +18,19 @@ const ( providerCredential = "secretname" ) -type testHealthServer struct { - Port int -} - -func (s *testHealthServer) Start(ctx context.Context) error { - mux := http.NewServeMux() - - endpoint := func(expectedCode int) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(expectedCode) - } - } - - mux.HandleFunc("/healthy", endpoint(200)) - mux.HandleFunc("/unhealthy", endpoint(500)) - - errCh := make(chan error) - - go func() { - errCh <- http.ListenAndServe(fmt.Sprintf(":%d", s.Port), mux) - }() - - select { - case <-ctx.Done(): - return ctx.Err() - case err := <-errCh: - return err +func testBuildManagedZone(name, ns, domainName string) *kuadrantdnsv1alpha1.ManagedZone { + return &kuadrantdnsv1alpha1.ManagedZone{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: kuadrantdnsv1alpha1.ManagedZoneSpec{ + ID: "1234", + DomainName: domainName, + Description: domainName, + SecretRef: kuadrantdnsv1alpha1.ProviderRef{ + Name: "secretname", + }, + }, } } diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 8b754daf..f4c34b64 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -22,10 +22,15 @@ import ( "context" "path/filepath" "testing" + "time" + "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -144,3 +149,22 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) + +func CreateNamespace(namespace *string) { + var generatedTestNamespace = "test-namespace-" + uuid.New().String() + + nsObject := &v1.Namespace{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Namespace"}, + ObjectMeta: metav1.ObjectMeta{Name: generatedTestNamespace}, + } + + err := k8sClient.Create(context.Background(), nsObject) + Expect(err).ToNot(HaveOccurred()) + + existingNamespace := &v1.Namespace{} + Eventually(func() error { + return k8sClient.Get(context.Background(), types.NamespacedName{Name: generatedTestNamespace}, existingNamespace) + }, time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + + *namespace = existingNamespace.Name +} diff --git a/test/e2e/single_cluster_record_test.go b/test/e2e/single_cluster_record_test.go index 828c4680..9506ad3b 100644 --- a/test/e2e/single_cluster_record_test.go +++ b/test/e2e/single_cluster_record_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/gomega/gstruct" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" externaldnsendpoint "sigs.k8s.io/external-dns/endpoint" @@ -99,6 +100,22 @@ var _ = Describe("Single Cluster Record Test", func() { g.Expect(err).NotTo(HaveOccurred()) g.Expect(ips).To(ContainElement(testTargetIP)) }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) + + By("ensuring zone records are created as expected") + testProvider, err := providerForManagedZone(ctx, testManagedZone) + Expect(err).NotTo(HaveOccurred()) + zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) + Expect(err).NotTo(HaveOccurred()) + Expect(zoneEndpoints).To(HaveLen(1)) + Expect(zoneEndpoints).To(ContainElements( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf(testTargetIP), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + })), + )) }) }) @@ -108,12 +125,12 @@ var _ = Describe("Single Cluster Record Test", func() { testTargetIP := "127.0.0.1" klbHostName := "klb." + testHostname - geo1KlbHostName := geoCode + "." + klbHostName + geo1KlbHostName := strings.ToLower(geoCode) + "." + klbHostName cluster1KlbHostName := "cluster1." + klbHostName dnsRecord = &v1alpha1.DNSRecord{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-record", + Name: testID, Namespace: testNamespace, }, Spec: v1alpha1.DNSRecordSpec{ @@ -207,6 +224,464 @@ var _ = Describe("Single Cluster Record Test", func() { g.Expect(err).NotTo(HaveOccurred()) g.Expect(ips).To(ContainElement(testTargetIP)) }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) + + By("ensuring zone records are created as expected") + testProvider, err := providerForManagedZone(ctx, testManagedZone) + Expect(err).NotTo(HaveOccurred()) + zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) + Expect(err).NotTo(HaveOccurred()) + if testDNSProvider == "gcp" { + Expect(zoneEndpoints).To(HaveLen(4)) + Expect(zoneEndpoints).To(ContainElements( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(cluster1KlbHostName), + "Targets": ConsistOf(testTargetIP), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf(klbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(geo1KlbHostName), + "Targets": ConsistOf(cluster1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "routingpolicy", Value: "weighted"}, + {Name: cluster1KlbHostName, Value: "200"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(klbHostName), + "Targets": ConsistOf(geo1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "routingpolicy", Value: "geo"}, + {Name: geo1KlbHostName, Value: geoCode}, + }), + })), + )) + } + if testDNSProvider == "aws" { + Expect(zoneEndpoints).To(HaveLen(5)) + Expect(zoneEndpoints).To(ContainElements( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(cluster1KlbHostName), + "Targets": ConsistOf(testTargetIP), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf(klbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(geo1KlbHostName), + "Targets": ConsistOf(cluster1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(cluster1KlbHostName), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "alias", Value: "false"}, + {Name: "aws/weight", Value: "200"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(klbHostName), + "Targets": ConsistOf(geo1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(geoCode), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "alias", Value: "false"}, + {Name: "aws/geolocation-country-code", Value: "US"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(klbHostName), + "Targets": ConsistOf(geo1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal("default"), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "alias", Value: "false"}, + {Name: "aws/geolocation-country-code", Value: "*"}, + }), + })), + )) + } + + }) + }) + + Context("with ownerID", func() { + Context("simple", func() { + It("makes available a hostname that can be resolved", func(ctx SpecContext) { + By("creating a dns record") + testTargetIP := "127.0.0.1" + dnsRecord = &v1alpha1.DNSRecord{ + ObjectMeta: metav1.ObjectMeta{ + Name: testID, + Namespace: testNamespace, + }, + Spec: v1alpha1.DNSRecordSpec{ + ManagedZoneRef: &v1alpha1.ManagedZoneReference{ + Name: testManagedZoneName, + }, + Endpoints: []*externaldnsendpoint.Endpoint{ + { + DNSName: testHostname, + Targets: []string{ + testTargetIP, + }, + RecordType: "A", + RecordTTL: 60, + }, + }, + OwnerID: ptr.To("test-owner"), + }, + } + err := k8sClient.Create(ctx, dnsRecord) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega, ctx context.Context) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(dnsRecord.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(conditions.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + })), + ) + }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) + + By("ensuring the authoritative nameserver resolves the hostname") + // speed up things by using the authoritative nameserver + authoritativeResolver := ResolverForDomainName(testZoneDomainName) + Eventually(func(g Gomega, ctx context.Context) { + ips, err := authoritativeResolver.LookupHost(ctx, testHostname) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ips).To(ContainElement(testTargetIP)) + }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) + + By("ensuring zone records are created as expected") + testProvider, err := providerForManagedZone(ctx, testManagedZone) + Expect(err).NotTo(HaveOccurred()) + zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) + Expect(err).NotTo(HaveOccurred()) + + Expect(zoneEndpoints).To(HaveLen(2)) + Expect(zoneEndpoints).To(ContainElements( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf(testTargetIP), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-a-" + testHostname), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + )) + }) + }) + + Context("loadbalanced", func() { + It("makes available a hostname that can be resolved", func(ctx SpecContext) { + By("creating a dns record") + testTargetIP := "127.0.0.1" + + klbHostName := "klb." + testHostname + geo1KlbHostName := strings.ToLower(geoCode) + "." + klbHostName + cluster1KlbHostName := "cluster1." + klbHostName + + dnsRecord = &v1alpha1.DNSRecord{ + ObjectMeta: metav1.ObjectMeta{ + Name: testID, + Namespace: testNamespace, + }, + Spec: v1alpha1.DNSRecordSpec{ + ManagedZoneRef: &v1alpha1.ManagedZoneReference{ + Name: testManagedZoneName, + }, + Endpoints: []*externaldnsendpoint.Endpoint{ + { + DNSName: cluster1KlbHostName, + Targets: []string{ + testTargetIP, + }, + RecordType: "A", + RecordTTL: 60, + }, + { + DNSName: testHostname, + Targets: []string{ + klbHostName, + }, + RecordType: "CNAME", + RecordTTL: 300, + }, + { + DNSName: geo1KlbHostName, + Targets: []string{ + cluster1KlbHostName, + }, + RecordType: "CNAME", + RecordTTL: 60, + SetIdentifier: cluster1KlbHostName, + ProviderSpecific: externaldnsendpoint.ProviderSpecific{ + { + Name: "weight", + Value: "200", + }, + }, + }, + { + DNSName: klbHostName, + Targets: []string{ + geo1KlbHostName, + }, + RecordType: "CNAME", + RecordTTL: 300, + SetIdentifier: geoCode, + ProviderSpecific: externaldnsendpoint.ProviderSpecific{ + { + Name: "geo-code", + Value: geoCode, + }, + }, + }, + { + DNSName: klbHostName, + Targets: []string{ + geo1KlbHostName, + }, + RecordType: "CNAME", + RecordTTL: 300, + SetIdentifier: "default", + ProviderSpecific: externaldnsendpoint.ProviderSpecific{ + { + Name: "geo-code", + Value: "*", + }, + }, + }, + }, + OwnerID: ptr.To("test-owner"), + }, + } + err := k8sClient.Create(ctx, dnsRecord) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega, ctx context.Context) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(dnsRecord.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(conditions.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + })), + ) + }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) + + By("ensuring the authoritative nameserver resolves the hostname") + // speed up things by using the authoritative nameserver + authoritativeResolver := ResolverForDomainName(testZoneDomainName) + Eventually(func(g Gomega, ctx context.Context) { + ips, err := authoritativeResolver.LookupHost(ctx, testHostname) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ips).To(ContainElement(testTargetIP)) + }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) + + By("ensuring zone records are created as expected") + testProvider, err := providerForManagedZone(ctx, testManagedZone) + Expect(err).NotTo(HaveOccurred()) + zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) + Expect(err).NotTo(HaveOccurred()) + if testDNSProvider == "gcp" { + Expect(zoneEndpoints).To(HaveLen(8)) + Expect(zoneEndpoints).To(ContainElements( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(cluster1KlbHostName), + "Targets": ConsistOf(testTargetIP), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf(klbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(geo1KlbHostName), + "Targets": ConsistOf(cluster1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "routingpolicy", Value: "weighted"}, + {Name: cluster1KlbHostName, Value: "200"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(klbHostName), + "Targets": ConsistOf(geo1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "routingpolicy", Value: "geo"}, + {Name: geo1KlbHostName, Value: geoCode}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-a-" + cluster1KlbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + testHostname), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + geo1KlbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + klbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + )) + } + if testDNSProvider == "aws" { + Expect(zoneEndpoints).To(HaveLen(10)) + Expect(zoneEndpoints).To(ContainElements( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(cluster1KlbHostName), + "Targets": ConsistOf(testTargetIP), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf(klbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(geo1KlbHostName), + "Targets": ConsistOf(cluster1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(cluster1KlbHostName), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "alias", Value: "false"}, + {Name: "aws/weight", Value: "200"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(klbHostName), + "Targets": ConsistOf(geo1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(geoCode), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "alias", Value: "false"}, + {Name: "aws/geolocation-country-code", Value: "US"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(klbHostName), + "Targets": ConsistOf(geo1KlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal("default"), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "alias", Value: "false"}, + {Name: "aws/geolocation-country-code", Value: "*"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-a-" + cluster1KlbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + testHostname), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + geo1KlbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(cluster1KlbHostName), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "aws/weight", Value: "200"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + klbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(geoCode), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "aws/geolocation-country-code", Value: "US"}, + }), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + klbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=test-owner\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal("default"), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "aws/geolocation-country-code", Value: "*"}, + }), + })), + )) + } + + }) }) }) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 02f39cc1..8b14dbac 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -9,6 +9,7 @@ import ( "math/big" "net" "os" + "regexp" "slices" "strings" "testing" @@ -19,12 +20,18 @@ import ( . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + externaldnsendpoint "sigs.k8s.io/external-dns/endpoint" + externaldnsprovider "sigs.k8s.io/external-dns/provider" "github.com/kuadrant/dns-operator/api/v1alpha1" + "github.com/kuadrant/dns-operator/internal/provider" + _ "github.com/kuadrant/dns-operator/internal/provider/aws" + _ "github.com/kuadrant/dns-operator/internal/provider/google" ) const ( @@ -45,6 +52,7 @@ var ( testNamespace string testDNSProvider string supportedProviders = []string{"aws", "gcp"} + testManagedZone *v1alpha1.ManagedZone ) func TestAPIs(t *testing.T) { @@ -71,6 +79,10 @@ var _ = BeforeSuite(func(ctx SpecContext) { Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) + testManagedZone = &v1alpha1.ManagedZone{} + err = k8sClient.Get(ctx, client.ObjectKey{Namespace: testNamespace, Name: testManagedZoneName}, testManagedZone) + Expect(err).NotTo(HaveOccurred()) + testSuiteID = "dns-op-e2e-" + GenerateName() }) @@ -113,3 +125,39 @@ func GenerateName() string { nBig, _ := rand.Int(rand.Reader, big.NewInt(1000000)) return namegenerator.NewNameGenerator(nBig.Int64()).Generate() } + +func EndpointsForHost(ctx context.Context, provider provider.Provider, host string) ([]*externaldnsendpoint.Endpoint, error) { + filtered := []*externaldnsendpoint.Endpoint{} + + records, err := provider.Records(ctx) + if err != nil { + return nil, err + } + + hostRegexp, err := regexp.Compile(host) + if err != nil { + return nil, err + } + + domainFilter := externaldnsendpoint.NewRegexDomainFilter(hostRegexp, nil) + + for _, record := range records { + // Ignore records that do not match the domain filter provided + if !domainFilter.Match(record.DNSName) { + GinkgoWriter.Printf("[debug] ignoring record %s that does not match domain filter %s\n", record.DNSName, host) + continue + } + filtered = append(filtered, record) + } + return filtered, nil +} + +func providerForManagedZone(ctx context.Context, mz *v1alpha1.ManagedZone) (provider.Provider, error) { + providerFactory := provider.NewFactory(k8sClient) + providerConfig := provider.Config{ + DomainFilter: externaldnsendpoint.NewDomainFilter([]string{mz.Spec.DomainName}), + ZoneTypeFilter: externaldnsprovider.NewZoneTypeFilter(""), + ZoneIDFilter: externaldnsprovider.NewZoneIDFilter([]string{mz.Status.ID}), + } + return providerFactory.ProviderFor(ctx, mz, providerConfig) +} From 00b66211803a8eb2f6b25cd4bf16fc5d101aa7da Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Mon, 11 Mar 2024 16:23:13 +0000 Subject: [PATCH 3/3] local-setup: Add local cleanup make target Add `local-cleanup` make target which deletes the development kind cluster created by `make local-setup`. --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index c08e2113..47718113 100644 --- a/Makefile +++ b/Makefile @@ -169,6 +169,10 @@ local-setup: $(KIND) ## Setup local development kind cluster, dependencies and o $(KUBECTL) -n ${TEST_NAMESPACE} get managedzones @echo "local-setup: Complete!!" +.PHONY: local-cleanup +local-cleanup: ## Delete local cluster + $(MAKE) kind-delete-cluster + .PHONY: local-deploy local-deploy: docker-build kind-load-image ## Deploy the dns operator into local kind cluster from the current code $(KUBECTL) config use-context kind-$(KIND_CLUSTER_NAME)