Skip to content

Commit

Permalink
aws health checks
Browse files Browse the repository at this point in the history
  • Loading branch information
philbrookes committed Apr 17, 2024
1 parent 84f6a59 commit a7155a8
Show file tree
Hide file tree
Showing 16 changed files with 732 additions and 12 deletions.
10 changes: 9 additions & 1 deletion api/v1alpha1/dnsrecord_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,15 @@ type HealthCheckSpec struct {
}

type HealthCheckStatus struct {
Conditions []metav1.Condition `json:"conditions,omitempty"`
Conditions []metav1.Condition `json:"conditions,omitempty"`
Probes []HealthCheckStatusProbe `json:"probes,omitempty"`
}

type HealthCheckStatusProbe struct {
ID string `json:"id"`
IPAddress string `json:"ipAddress"`
Host string `json:"host"`
Synced bool `json:"synced,omitempty"`
}

// DNSRecordSpec defines the desired state of DNSRecord
Expand Down
20 changes: 20 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 17 additions & 0 deletions config/crd/bases/kuadrant.io_dnsrecords.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,23 @@ spec:
- type
type: object
type: array
probes:
items:
properties:
host:
type: string
id:
type: string
ipAddress:
type: string
synced:
type: boolean
required:
- host
- id
- ipAddress
type: object
type: array
type: object
observedGeneration:
description: observedGeneration is the most recently observed generation
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ require (
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,8 @@ github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rubenv/sql-migrate v0.0.0-20200212082348-64f95ea68aa3/go.mod h1:rtQlpHw+eR6UrqaS3kX1VYeaCxzCVdimDS7g5Ln4pPc=
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
Expand Down
34 changes: 27 additions & 7 deletions internal/controller/dnsrecord_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,11 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
// success
dnsRecord.Status.ObservedGeneration = dnsRecord.Generation
dnsRecord.Status.Endpoints = dnsRecord.Spec.Endpoints

if err := r.ReconcileHealthChecks(ctx, dnsRecord); err != nil {
return ctrl.Result{}, err
}

return r.updateStatus(ctx, previous, dnsRecord, requeueAfter)
}

Expand Down Expand Up @@ -250,6 +255,27 @@ func setDNSRecordCondition(dnsRecord *v1alpha1.DNSRecord, conditionType string,
meta.SetStatusCondition(&dnsRecord.Status.Conditions, cond)
}

func (r *DNSRecordReconciler) getDNSProvider(ctx context.Context, dnsRecord *v1alpha1.DNSRecord) (provider.Provider, error) {
managedZone := &v1alpha1.ManagedZone{
ObjectMeta: metav1.ObjectMeta{
Name: dnsRecord.Spec.ManagedZoneRef.Name,
Namespace: dnsRecord.Namespace,
},
}
err := r.Get(ctx, client.ObjectKeyFromObject(managedZone), managedZone, &client.GetOptions{})
if err != nil {
return nil, err
}

providerConfig := provider.Config{
DomainFilter: externaldnsendpoint.NewDomainFilter([]string{managedZone.Spec.DomainName}),
ZoneTypeFilter: externaldnsprovider.NewZoneTypeFilter(""),
ZoneIDFilter: externaldnsprovider.NewZoneIDFilter([]string{managedZone.Status.ID}),
}

return r.ProviderFactory.ProviderFor(ctx, managedZone, providerConfig)
}

func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alpha1.DNSRecord, managedZone *v1alpha1.ManagedZone, isDelete bool) (time.Duration, error) {
logger := log.FromContext(ctx)
filterDomain, _ := strings.CutPrefix(managedZone.Spec.DomainName, v1alpha1.WildcardPrefix)
Expand All @@ -258,13 +284,7 @@ func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alp
}
rootDomainFilter := externaldnsendpoint.NewDomainFilter([]string{filterDomain})

providerConfig := provider.Config{
DomainFilter: externaldnsendpoint.NewDomainFilter([]string{managedZone.Spec.DomainName}),
ZoneTypeFilter: externaldnsprovider.NewZoneTypeFilter(""),
ZoneIDFilter: externaldnsprovider.NewZoneIDFilter([]string{managedZone.Status.ID}),
}
logger.V(3).Info("applyChanges", "zone", managedZone.Spec.DomainName, "rootDomainFilter", rootDomainFilter, "providerConfig", providerConfig)
dnsProvider, err := r.ProviderFactory.ProviderFor(ctx, managedZone, providerConfig)
dnsProvider, err := r.getDNSProvider(ctx, dnsRecord)
if err != nil {
return noRequeueDuration, err
}
Expand Down
198 changes: 198 additions & 0 deletions internal/controller/dnsrecord_healthchecks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
package controller

import (
"context"
"crypto/md5"
"fmt"
"io"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
externaldns "sigs.k8s.io/external-dns/endpoint"

"github.com/kuadrant/dns-operator/api/v1alpha1"
"github.com/kuadrant/dns-operator/internal/provider"
)

// healthChecksConfig represents the user configuration for the health checks
type healthChecksConfig struct {
Endpoint string
Port *int64
FailureThreshold *int64
Protocol *provider.HealthCheckProtocol
}

func (r *DNSRecordReconciler) ReconcileHealthChecks(ctx context.Context, dnsRecord *v1alpha1.DNSRecord) error {
var results []provider.HealthCheckResult
var err error

dnsProvider, err := r.getDNSProvider(ctx, dnsRecord)
if err != nil {
return err
}

healthCheckReconciler := dnsProvider.HealthCheckReconciler()

// Get the configuration for the health checks. If no configuration is
// set, ensure that the health checks are deleted
config := getHealthChecksConfig(dnsRecord)

if config == nil {
// deleting
for _, endpoint := range dnsRecord.Spec.Endpoints {
addresses := provider.GetExternalAddresses(endpoint, dnsRecord)
for _, address := range addresses {
probeStatus := r.getProbeStatus(address, dnsRecord)
if probeStatus == nil {
continue
}
result, err := healthCheckReconciler.Delete(ctx, endpoint, probeStatus)
if err != nil {
return err
}

results = append(results, result)
}
}
} else {
for _, dnsEndpoint := range dnsRecord.Spec.Endpoints {
addresses := provider.GetExternalAddresses(dnsEndpoint, dnsRecord)
for _, address := range addresses {
endpointId, err := idForEndpoint(dnsRecord, dnsEndpoint, address)
if err != nil {
return err
}

spec := provider.HealthCheckSpec{
Id: endpointId,
Name: fmt.Sprintf("%s-%s-%s", *dnsRecord.Spec.RootHost, dnsEndpoint.DNSName, address),
Host: dnsRecord.Spec.RootHost,
Path: config.Endpoint,
Port: config.Port,
Protocol: config.Protocol,
FailureThreshold: config.FailureThreshold,
}

probeStatus := r.getProbeStatus(address, dnsRecord)

result, err := healthCheckReconciler.Reconcile(ctx, spec, dnsEndpoint, probeStatus, address)
if err != nil {
return err
}
results = append(results, result)
}
}
}

result := r.reconcileHealthCheckStatus(results, dnsRecord)
return result
}

func (r *DNSRecordReconciler) getProbeStatus(address string, dnsRecord *v1alpha1.DNSRecord) *v1alpha1.HealthCheckStatusProbe {
if dnsRecord.Status.HealthCheck == nil || dnsRecord.Status.HealthCheck.Probes == nil {
return nil
}
for _, probeStatus := range dnsRecord.Status.HealthCheck.Probes {
if probeStatus.IPAddress == address {
return &probeStatus
}
}

return nil
}

func (r *DNSRecordReconciler) reconcileHealthCheckStatus(results []provider.HealthCheckResult, dnsRecord *v1alpha1.DNSRecord) error {
probesCondition := metav1.Condition{
ObservedGeneration: dnsRecord.Generation,
Status: metav1.ConditionTrue,
Reason: "AllProbesSynced",
LastTransitionTime: metav1.Now(),
Message: fmt.Sprintf("all %v probes synced successfully", len(results)),
Type: "healthProbesSynced",
}

var allSynced = metav1.ConditionTrue

if dnsRecord.Status.HealthCheck == nil {
dnsRecord.Status.HealthCheck = &v1alpha1.HealthCheckStatus{
Conditions: []metav1.Condition{},
Probes: []v1alpha1.HealthCheckStatusProbe{},
}
}

for _, c := range dnsRecord.Status.HealthCheck.Conditions {
if c.Type == "healthProbesSynced" {
probesCondition = c
}
}
if probesCondition.ObservedGeneration != dnsRecord.Generation {
probesCondition.ObservedGeneration = dnsRecord.Generation
probesCondition.LastTransitionTime = metav1.Now()
}

dnsRecord.Status.HealthCheck.Probes = []v1alpha1.HealthCheckStatusProbe{}

for _, result := range results {
status := true
if result.Result == provider.HealthCheckFailed {
status = false
allSynced = metav1.ConditionFalse
}

dnsRecord.Status.HealthCheck.Probes = append(dnsRecord.Status.HealthCheck.Probes, v1alpha1.HealthCheckStatusProbe{
ID: result.ID,
IPAddress: result.IPAddress,
Host: result.Host,
Synced: status,
})
}

if probesCondition.Status != allSynced {
probesCondition.Status = allSynced
probesCondition.LastTransitionTime = metav1.Now()
if allSynced == metav1.ConditionTrue {
probesCondition.Reason = "AllProbesSynced"
} else {
probesCondition.Reason = "UnsyncedProbes"
probesCondition.Message = "some probes have not yet successfully synced to the DNS Provider"
}
}

dnsRecord.Status.HealthCheck.Conditions = []metav1.Condition{probesCondition}

return nil
}

func getHealthChecksConfig(policy *v1alpha1.DNSRecord) *healthChecksConfig {
if policy.Spec.HealthCheck == nil {
return nil
}

return &healthChecksConfig{
Endpoint: policy.Spec.HealthCheck.Endpoint,
Port: valueAs(toInt64, policy.Spec.HealthCheck.Port),
FailureThreshold: valueAs(toInt64, policy.Spec.HealthCheck.FailureThreshold),
Protocol: (*provider.HealthCheckProtocol)(policy.Spec.HealthCheck.Protocol),
}
}

func valueAs[T, R any](f func(T) R, original *T) *R {
if original == nil {
return nil
}

value := f(*original)
return &value
}

func toInt64(original int) int64 {
return int64(original)
}

// idForEndpoint returns a unique identifier for an endpoint
func idForEndpoint(dnsRecord *v1alpha1.DNSRecord, endpoint *externaldns.Endpoint, address string) (string, error) {
hash := md5.New()
if _, err := io.WriteString(hash, fmt.Sprintf("%s/%s@%s:%s", dnsRecord.Name, endpoint.SetIdentifier, endpoint.DNSName, address)); err != nil {
return "", fmt.Errorf("unexpected error creating ID for endpoint %s", endpoint.SetIdentifier)
}
return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
24 changes: 20 additions & 4 deletions internal/provider/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
)

const (
ProviderSpecificHealthCheckID = "aws/health-check-id"
providerSpecificWeight = "aws/weight"
providerSpecificGeolocationCountryCode = "aws/geolocation-country-code"
providerSpecificGeolocationContinentCode = "aws/geolocation-continent-code"
Expand All @@ -50,10 +51,11 @@ const (

type Route53DNSProvider struct {
*externaldnsprovideraws.AWSProvider
awsConfig externaldnsprovideraws.AWSConfig
logger logr.Logger
route53Client *route53.Route53
ctx context.Context
awsConfig externaldnsprovideraws.AWSConfig
logger logr.Logger
route53Client *route53.Route53
ctx context.Context
healthCheckReconciler provider.HealthCheckReconciler
}

var _ provider.Provider = &Route53DNSProvider{}
Expand Down Expand Up @@ -108,6 +110,13 @@ func NewProviderFromSecret(ctx context.Context, s *v1.Secret, c provider.Config)
}

// #### External DNS Provider ####
func (p *Route53DNSProvider) HealthCheckReconciler() provider.HealthCheckReconciler {
if p.healthCheckReconciler == nil {
p.healthCheckReconciler = NewRoute53HealthCheckReconciler(p.route53Client)
}

return p.healthCheckReconciler
}

func (p *Route53DNSProvider) AdjustEndpoints(endpoints []*externaldnsendpoint.Endpoint) ([]*externaldnsendpoint.Endpoint, error) {
endpoints, err := p.AWSProvider.AdjustEndpoints(endpoints)
Expand Down Expand Up @@ -231,6 +240,13 @@ func (p *Route53DNSProvider) DeleteManagedZone(zone *v1alpha1.ManagedZone) error
return nil
}

func (*Route53DNSProvider) ProviderSpecific() provider.ProviderSpecificLabels {
return provider.ProviderSpecificLabels{
Weight: providerSpecificWeight,
HealthCheckID: ProviderSpecificHealthCheckID,
}
}

// Register this Provider with the provider factory
func init() {
provider.RegisterProvider("aws", NewProviderFromSecret)
Expand Down
Loading

0 comments on commit a7155a8

Please sign in to comment.