From 1376eb618143fbf63253d604edf0e7e96a120859 Mon Sep 17 00:00:00 2001 From: Phil Brookes Date: Wed, 24 Apr 2024 09:38:11 +0200 Subject: [PATCH] adding health check e2e tests --- internal/controller/dnsrecord_healthchecks.go | 2 +- internal/provider/aws/health.go | 5 + internal/provider/cached.go | 4 + internal/provider/fake.go | 3 + internal/provider/google/health.go | 4 + internal/provider/health.go | 1 + .../geo-dnsrecord-healthchecks.yaml | 53 +++++ test/e2e/healthcheck_test.go | 215 ++++++++++++++++++ test/e2e/helpers/fixtures.go | 38 ++++ test/e2e/suite_test.go | 22 +- 10 files changed, 340 insertions(+), 7 deletions(-) create mode 100644 test/e2e/fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml create mode 100644 test/e2e/healthcheck_test.go create mode 100644 test/e2e/helpers/fixtures.go diff --git a/internal/controller/dnsrecord_healthchecks.go b/internal/controller/dnsrecord_healthchecks.go index 16877df..00cd01a 100644 --- a/internal/controller/dnsrecord_healthchecks.go +++ b/internal/controller/dnsrecord_healthchecks.go @@ -174,7 +174,7 @@ func getHealthChecksConfig(dnsRecord *v1alpha1.DNSRecord) *healthChecksConfig { // idForEndpoint returns a unique identifier for an endpoint func idForEndpoint(dnsRecord *v1alpha1.DNSRecord, endpoint *externaldns.Endpoint, address string) (string, error) { hash := md5.New() - if _, err := io.WriteString(hash, fmt.Sprintf("%s/%s@%s:%s", dnsRecord.Name, endpoint.SetIdentifier, endpoint.DNSName, address)); err != nil { + if _, err := io.WriteString(hash, fmt.Sprintf("%s/%s@%s:%s-%v", dnsRecord.Name, endpoint.SetIdentifier, endpoint.DNSName, address, dnsRecord.Generation)); err != nil { return "", fmt.Errorf("unexpected error creating ID for endpoint %s", endpoint.SetIdentifier) } return fmt.Sprintf("%x", hash.Sum(nil)), nil diff --git a/internal/provider/aws/health.go b/internal/provider/aws/health.go index 06108b3..68f9816 100644 --- a/internal/provider/aws/health.go +++ b/internal/provider/aws/health.go @@ -52,6 +52,11 @@ func getTransitionTime(probeConditions []metav1.Condition, conditionType string, return metav1.Now() } +func (r *Route53HealthCheckReconciler) HealthCheckExists(ctx context.Context, probeStatus *v1alpha1.HealthCheckStatusProbe) (bool, error) { + _, exists, err := r.findHealthCheck(ctx, probeStatus) + return exists, err +} + func (r *Route53HealthCheckReconciler) Reconcile(ctx context.Context, spec provider.HealthCheckSpec, endpoint *externaldns.Endpoint, probeStatus *v1alpha1.HealthCheckStatusProbe, address string) provider.HealthCheckResult { healthCheck, exists, err := r.findHealthCheck(ctx, probeStatus) if err != nil { diff --git a/internal/provider/cached.go b/internal/provider/cached.go index e7557f0..1cde53d 100644 --- a/internal/provider/cached.go +++ b/internal/provider/cached.go @@ -28,6 +28,10 @@ func NewCachedHealthCheckReconciler(provider Provider, reconciler HealthCheckRec } } +func (r *CachedHealthCheckReconciler) HealthCheckExists(ctx context.Context, probeStatus *v1alpha1.HealthCheckStatusProbe) (bool, error) { + return r.reconciler.HealthCheckExists(ctx, probeStatus) +} + // Delete implements HealthCheckReconciler func (r *CachedHealthCheckReconciler) Delete(ctx context.Context, endpoint *externaldns.Endpoint, probeStatus *v1alpha1.HealthCheckStatusProbe) (HealthCheckResult, error) { id, ok := r.getHealthCheckID(endpoint) diff --git a/internal/provider/fake.go b/internal/provider/fake.go index d28d88a..c18c14c 100644 --- a/internal/provider/fake.go +++ b/internal/provider/fake.go @@ -11,6 +11,9 @@ import ( type FakeHealthCheckReconciler struct{} +func (*FakeHealthCheckReconciler) HealthCheckExists(ctx context.Context, probeStatus *v1alpha1.HealthCheckStatusProbe) (bool, error) { + return true, nil +} func (*FakeHealthCheckReconciler) Reconcile(_ context.Context, _ HealthCheckSpec, _ *externaldns.Endpoint, _ *v1alpha1.HealthCheckStatusProbe, _ string) HealthCheckResult { return HealthCheckResult{HealthCheckCreated, "fakeID", "", "", metav1.Condition{}} } diff --git a/internal/provider/google/health.go b/internal/provider/google/health.go index f8d6b00..f636365 100644 --- a/internal/provider/google/health.go +++ b/internal/provider/google/health.go @@ -18,6 +18,10 @@ func NewGCPHealthCheckReconciler() *GCPHealthCheckReconciler { return &GCPHealthCheckReconciler{} } +func (r *GCPHealthCheckReconciler) HealthCheckExists(_ context.Context, _ *v1alpha1.HealthCheckStatusProbe) (bool, error) { + return true, nil +} + func (r *GCPHealthCheckReconciler) Reconcile(_ context.Context, _ provider.HealthCheckSpec, _ *externaldns.Endpoint, _ *v1alpha1.HealthCheckStatusProbe, _ string) provider.HealthCheckResult { return provider.HealthCheckResult{} } diff --git a/internal/provider/health.go b/internal/provider/health.go index 8b8e190..de9e8f9 100644 --- a/internal/provider/health.go +++ b/internal/provider/health.go @@ -12,6 +12,7 @@ import ( type HealthCheckReconciler interface { Reconcile(ctx context.Context, spec HealthCheckSpec, endpoint *externaldns.Endpoint, probeStatus *v1alpha1.HealthCheckStatusProbe, address string) HealthCheckResult Delete(ctx context.Context, endpoint *externaldns.Endpoint, probeStatus *v1alpha1.HealthCheckStatusProbe) (HealthCheckResult, error) + HealthCheckExists(ctx context.Context, probeStatus *v1alpha1.HealthCheckStatusProbe) (bool, error) } type HealthCheckSpec struct { diff --git a/test/e2e/fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml b/test/e2e/fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml new file mode 100644 index 0000000..ffd3ab8 --- /dev/null +++ b/test/e2e/fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml @@ -0,0 +1,53 @@ +apiVersion: kuadrant.io/v1alpha1 +kind: DNSRecord +metadata: + name: ${testID} + namespace: ${testNamespace} +spec: + healthCheck: + endpoint: "/" + port: 80 + protocol: "HTTPS" + failureThreshold: 3 + endpoints: + - dnsName: 14byhk-2k52h1.klb.${testHostname} + recordTTL: 60 + recordType: A + targets: + - 172.32.200.1 + - dnsName: ${testHostname} + recordTTL: 300 + recordType: CNAME + targets: + - klb.${testHostname} + - dnsName: eu.klb.${testHostname} + providerSpecific: + - name: weight + value: "120" + recordTTL: 60 + recordType: CNAME + setIdentifier: 14byhk-2k52h1.klb.${testHostname} + targets: + - 14byhk-2k52h1.klb.${testHostname} + - dnsName: klb.${testHostname} + providerSpecific: + - name: geo-code + value: ${testGeoCode} + recordTTL: 300 + recordType: CNAME + setIdentifier: ${testGeoCode} + targets: + - eu.klb.${testHostname} + - dnsName: klb.${testHostname} + providerSpecific: + - name: geo-code + value: '*' + recordTTL: 300 + recordType: CNAME + setIdentifier: default + targets: + - eu.klb.${testHostname} + managedZone: + name: ${TEST_DNS_MANAGED_ZONE_NAME} + ownerID: 2bq03i + rootHost: ${testHostname} diff --git a/test/e2e/healthcheck_test.go b/test/e2e/healthcheck_test.go new file mode 100644 index 0000000..3ce64e4 --- /dev/null +++ b/test/e2e/healthcheck_test.go @@ -0,0 +1,215 @@ +//go:build e2e + +package e2e + +import ( + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/strings/slices" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kuadrant/dns-operator/api/v1alpha1" + "github.com/kuadrant/dns-operator/test/e2e/helpers" +) + +// Test Cases covering multiple creation and deletion of health checks +var _ = Describe("Health Check Test", Serial, func() { + // tests can be run in parallel in the same cluster + var testID string + // testDomainName generated domain for this test e.g. t-e2e-12345.e2e.hcpapps.net + var testDomainName string + // testHostname generated hostname for this test e.g. t-gw-mgc-12345.t-e2e-12345.e2e.hcpapps.net + var testHostname string + + var dnsRecord *v1alpha1.DNSRecord + + BeforeEach(func() { + testID = "t-health-" + GenerateName() + testDomainName = strings.Join([]string{testSuiteID, testZoneDomainName}, ".") + testHostname = strings.Join([]string{testID, testDomainName}, ".") + helpers.SetTestEnv("testID", testID) + helpers.SetTestEnv("testHostname", testHostname) + helpers.SetTestEnv("testNamespace", testNamespace) + }) + + AfterEach(func(ctx SpecContext) { + if dnsRecord != nil { + err := k8sClient.Delete(ctx, dnsRecord, + client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + } + }) + + Context("DNS Provider health checks", func() { + It("creates health checks for a health check spec", func(ctx SpecContext) { + healthChecksSupported := false + if slices.Contains(supportedHealthCheckProviders, strings.ToLower(testDNSProvider)) { + healthChecksSupported = true + } + + provider, err := providerForManagedZone(ctx, testManagedZone) + Expect(err).To(BeNil()) + + By("creating a DNS Record") + dnsRecord = &v1alpha1.DNSRecord{} + err = helpers.ResourceFromFile("./fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml", dnsRecord, helpers.GetTestEnv) + Expect(err).ToNot(HaveOccurred()) + + err = k8sClient.Create(ctx, dnsRecord) + Expect(err).ToNot(HaveOccurred()) + + By("Confirming the DNS Record status") + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).ToNot(HaveOccurred()) + + if healthChecksSupported { + g.Expect(dnsRecord.Status.HealthCheck).ToNot(BeNil()) + g.Expect(&dnsRecord.Status.HealthCheck.Probes).ToNot(BeNil()) + g.Expect(len(dnsRecord.Status.HealthCheck.Probes)).ToNot(BeZero()) + for _, condition := range dnsRecord.Status.HealthCheck.Conditions { + if condition.Type == "healthProbesSynced" { + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(condition.Reason).To(Equal("AllProbesSynced")) + } + } + } else { + g.Expect(dnsRecord.Status.HealthCheck).ToNot(BeNil()) + g.Expect(dnsRecord.Status.HealthCheck.Probes).To(BeNil()) + } + + for _, probe := range dnsRecord.Status.HealthCheck.Probes { + g.Expect(probe.Host).To(Equal(testHostname)) + g.Expect(probe.IPAddress).To(Equal("172.32.200.1")) + g.Expect(probe.ID).ToNot(Equal("")) + + for _, probeCondition := range probe.Conditions { + g.Expect(probeCondition.Type).To(Equal("ProbeSynced")) + g.Expect(probeCondition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(probeCondition.Message).To(ContainSubstring(fmt.Sprintf("id: %v, address: %v, host: %v", probe.ID, probe.IPAddress, probe.Host))) + } + } + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("confirming the health checks exist in the provider") + Eventually(func(g Gomega) { + if !healthChecksSupported { + g.Expect(len(dnsRecord.Status.HealthCheck.Probes)).To(BeZero()) + } + for _, healthCheck := range dnsRecord.Status.HealthCheck.Probes { + exists, err := provider.HealthCheckReconciler().HealthCheckExists(ctx, &healthCheck) + g.Expect(err).To(BeNil()) + g.Expect(exists).To(BeTrue()) + } + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("Removing the health check") + oldHealthCheckStatus := dnsRecord.Status.HealthCheck.DeepCopy() + Eventually(func(g Gomega) { + patchFrom := client.MergeFrom(dnsRecord.DeepCopy()) + dnsRecord.Spec.HealthCheck = nil + err := k8sClient.Patch(ctx, dnsRecord, patchFrom) + g.Expect(err).To(BeNil()) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("Confirming the DNS Record status") + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(dnsRecord.Status.HealthCheck).To(BeNil()) + }) + + By("confirming the health checks were removed in the provider") + Eventually(func(g Gomega) { + for _, healthCheck := range oldHealthCheckStatus.Probes { + exists, err := provider.HealthCheckReconciler().HealthCheckExists(ctx, &healthCheck) + g.Expect(err).NotTo(BeNil()) + g.Expect(exists).To(BeFalse()) + } + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("Adding a health check spec") + Eventually(func(g Gomega) { + patchFrom := client.MergeFrom(dnsRecord.DeepCopy()) + err = helpers.ResourceFromFile("./fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml", dnsRecord, helpers.GetTestEnv) + g.Expect(err).ToNot(HaveOccurred()) + err := k8sClient.Patch(ctx, dnsRecord, patchFrom) + g.Expect(err).To(BeNil()) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("Confirming the DNS Record status") + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).ToNot(HaveOccurred()) + + if healthChecksSupported { + g.Expect(dnsRecord.Status.HealthCheck).ToNot(BeNil()) + g.Expect(&dnsRecord.Status.HealthCheck.Probes).ToNot(BeNil()) + g.Expect(len(dnsRecord.Status.HealthCheck.Probes)).ToNot(BeZero()) + for _, condition := range dnsRecord.Status.HealthCheck.Conditions { + if condition.Type == "healthProbesSynced" { + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(condition.Reason).To(Equal("AllProbesSynced")) + } + } + } else { + g.Expect(dnsRecord.Status.HealthCheck).ToNot(BeNil()) + g.Expect(dnsRecord.Status.HealthCheck.Probes).To(BeNil()) + } + + for _, probe := range dnsRecord.Status.HealthCheck.Probes { + g.Expect(probe.Host).To(Equal(testHostname)) + g.Expect(probe.IPAddress).To(Equal("172.32.200.1")) + g.Expect(probe.ID).ToNot(Equal("")) + + for _, probeCondition := range probe.Conditions { + g.Expect(probeCondition.Type).To(Equal("ProbeSynced")) + g.Expect(probeCondition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(probeCondition.Message).To(ContainSubstring(fmt.Sprintf("id: %v, address: %v, host: %v", probe.ID, probe.IPAddress, probe.Host))) + } + } + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("confirming the health checks exist in the provider") + Eventually(func(g Gomega) { + if !healthChecksSupported { + g.Expect(len(dnsRecord.Status.HealthCheck.Probes)).To(BeZero()) + } + for _, healthCheck := range dnsRecord.Status.HealthCheck.Probes { + exists, err := provider.HealthCheckReconciler().HealthCheckExists(ctx, &healthCheck) + g.Expect(err).To(BeNil()) + g.Expect(exists).To(BeTrue()) + } + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("Deleting the DNS Record") + oldHealthCheckStatus = dnsRecord.Status.HealthCheck.DeepCopy() + err = helpers.ResourceFromFile("./fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml", dnsRecord, helpers.GetTestEnv) + Expect(err).ToNot(HaveOccurred()) + Eventually(func(g Gomega) { + err := k8sClient.Delete(ctx, dnsRecord) + g.Expect(client.IgnoreNotFound(err)).To(BeNil()) + + err = k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(errors.IsNotFound(err)).Should(BeTrue()) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + By("confirming the health checks were removed in the provider") + Eventually(func(g Gomega) { + for _, healthCheck := range oldHealthCheckStatus.Probes { + exists, err := provider.HealthCheckReconciler().HealthCheckExists(ctx, &healthCheck) + g.Expect(err).NotTo(BeNil()) + g.Expect(exists).To(BeFalse()) + } + }, TestTimeoutMedium, time.Second).Should(Succeed()) + + }) + }) +}) diff --git a/test/e2e/helpers/fixtures.go b/test/e2e/helpers/fixtures.go new file mode 100644 index 0000000..606b039 --- /dev/null +++ b/test/e2e/helpers/fixtures.go @@ -0,0 +1,38 @@ +package helpers + +import ( + "os" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes/scheme" +) + +var testEnvVars map[string]string + +func ResourceFromFile(file string, destObject runtime.Object, expandFunc func(string) string) error { + decode := serializer.NewCodecFactory(scheme.Scheme).UniversalDeserializer().Decode + stream, err := os.ReadFile(file) + if err != nil { + return err + } + stream = []byte(os.Expand(string(stream), expandFunc)) + _, _, err = decode(stream, nil, destObject) + return err +} + +func GetTestEnv(key string) string { + if testEnvVars != nil { + if v, ok := testEnvVars[key]; ok { + return v + } + } + return os.Getenv(key) +} + +func SetTestEnv(key, value string) { + if testEnvVars == nil { + testEnvVars = map[string]string{} + } + testEnvVars[key] = value +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 2be5f68..67cb2c5 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -34,6 +34,7 @@ import ( "github.com/kuadrant/dns-operator/internal/provider" _ "github.com/kuadrant/dns-operator/internal/provider/aws" _ "github.com/kuadrant/dns-operator/internal/provider/google" + "github.com/kuadrant/dns-operator/test/e2e/helpers" ) const ( @@ -42,6 +43,8 @@ const ( dnsManagedZoneName = "TEST_DNS_MANAGED_ZONE_NAME" dnsNamespace = "TEST_DNS_NAMESPACE" dnsProvider = "TEST_DNS_PROVIDER" + TestTimeoutMedium = 10 * time.Second + TestTimeoutLong = 60 * time.Second ) var ( @@ -49,12 +52,13 @@ var ( // testSuiteID is a randomly generated identifier for the test suite testSuiteID string // testZoneDomainName provided domain name for the testZoneID e.g. e2e.hcpapps.net - testZoneDomainName string - testManagedZoneName string - testNamespace string - testDNSProvider string - supportedProviders = []string{"aws", "gcp"} - testManagedZone *v1alpha1.ManagedZone + testZoneDomainName string + testManagedZoneName string + testNamespace string + testDNSProvider string + supportedProviders = []string{"aws", "gcp"} + supportedHealthCheckProviders = []string{"aws"} + testManagedZone *v1alpha1.ManagedZone ) func TestAPIs(t *testing.T) { @@ -89,6 +93,12 @@ var _ = BeforeSuite(func(ctx SpecContext) { Expect(err).NotTo(HaveOccurred()) testSuiteID = "dns-op-e2e-" + GenerateName() + + geoCode := "EU" + if testDNSProvider == "gcp" { + geoCode = "europe-west1" + } + helpers.SetTestEnv("testGeoCode", geoCode) }) func ResolverForDomainName(domainName string) *net.Resolver {