Skip to content

Commit

Permalink
e2e healthcheck test
Browse files Browse the repository at this point in the history
Signed-off-by: Maskym Vavilov <[email protected]>
  • Loading branch information
maleck13 authored and maksymvavilov committed Oct 31, 2024
1 parent a8d4411 commit 8cc4593
Show file tree
Hide file tree
Showing 18 changed files with 403 additions and 16 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/ci-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,16 @@ jobs:
- name: Setup environment
if: ${{ !inputs.ginkgoDryRun }}
run: |
make local-setup DEPLOY=true TEST_NAMESPACE=${{ env.TEST_NAMESPACE }}
make local-setup DEPLOY=true MOCK_HEALTH_CHECKS=true TEST_NAMESPACE=${{ env.TEST_NAMESPACE }}
kubectl -n ${{ env.TEST_NAMESPACE }} get secret/dns-provider-credentials-aws
kubectl -n ${{ env.TEST_NAMESPACE }} get secret/dns-provider-credentials-gcp
kubectl -n ${{ env.TEST_NAMESPACE }} get secret/dns-provider-credentials-azure
- name: Run healthcheck suite
run: |
export TEST_DNS_PROVIDER_SECRET_NAME=dns-provider-credentials-inmemory
export TEST_DNS_ZONE_DOMAIN_NAME=e2e.hcpapps.net
export TEST_DNS_NAMESPACES=${{ env.TEST_NAMESPACE }}
make test-e2e-healthchecks
- name: Run suite AWS
run: |
export TEST_DNS_PROVIDER_SECRET_NAME=dns-provider-credentials-aws
Expand Down
14 changes: 12 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ test-integration: manifests generate fmt vet envtest ginkgo ## Run integration t
test-e2e: ginkgo
$(GINKGO) $(GINKGO_FLAGS) -tags=e2e ./test/e2e

.PHONY: test-e2e-healthchecks
test-e2e-healthchecks: ginkgo
$(GINKGO) $(GINKGO_FLAGS) -tags=e2e_healthchecks ./test/e2e

.PHONY: test-e2e-multi
test-e2e-multi: ginkgo
$(GINKGO) $(GINKGO_FLAGS) -tags=e2e --label-filter=multi_record ./test/e2e
Expand Down Expand Up @@ -242,10 +246,11 @@ run: manifests generate fmt vet ## Run a controller from your host.
go run -ldflags "-X main.version=v${VERSION} -X main.gitSHA=${GIT_SHA} -X main.dirty=${DIRTY}" --race ./cmd/main.go --zap-devel --provider inmemory,aws,google,azure

.PHONY: run-with-probes
run-with-probes: MOCK_HEALTH_CHECKS=false
run-with-probes: GIT_SHA=$(shell git rev-parse HEAD || echo "unknown")
run-with-probes: DIRTY=$(shell hack/check-git-dirty.sh || echo "unknown")
run-with-probes: manifests generate fmt vet ## Run a controller from your host.
go run -ldflags "-X main.version=v${VERSION} -X main.gitSHA=${GIT_SHA} -X main.dirty=${DIRTY}" --race ./cmd/main.go --zap-devel --provider inmemory,aws,google,azure
go run -ldflags "-X main.version=v${VERSION} -X main.gitSHA=${GIT_SHA} -X main.dirty=${DIRTY}" --race ./cmd/main.go --zap-devel --provider inmemory,aws,google,azure --e2e-test-only-use-mock-health-transport=${MOCK_HEALTH_CHECKS}

# If you wish built the manager image targeting other platforms you can use the --platform flag.
# (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.
Expand Down Expand Up @@ -292,9 +297,14 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -

.PHONY: deploy
deploy: MOCK_HEALTH_CHECKS=false
deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
$(KUSTOMIZE) build config/deploy/local | $(KUBECTL) apply -f -
@if [ ${MOCK_HEALTH_CHECKS} = "false" ]; then\
$(KUSTOMIZE) build config/deploy/local | $(KUBECTL) apply -f - ;\
else\
$(KUSTOMIZE) build config/deploy/mock-transport | $(KUBECTL) apply -f - ;\
fi ;\

.PHONY: deploy-namespaced
deploy-namespaced: manifests kustomize generate-cluster-overlay ## Deploy controller to the K8s cluster specified in ~/.kube/config.
Expand Down
14 changes: 13 additions & 1 deletion bundle/manifests/dns-operator.clusterserviceversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ metadata:
capabilities: Basic Install
categories: Integration & Delivery
containerImage: quay.io/kuadrant/dns-operator:latest
createdAt: "2024-10-18T15:21:24Z"
createdAt: "2024-10-29T13:53:25Z"
description: A Kubernetes Operator to manage the lifecycle of DNS resources
operators.operatorframework.io/builder: operator-sdk-v1.33.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v4
Expand Down Expand Up @@ -90,6 +90,18 @@ spec:
spec:
clusterPermissions:
- rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
Expand Down
12 changes: 12 additions & 0 deletions charts/dns-operator/templates/manifests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -761,6 +761,18 @@ metadata:
app.kubernetes.io/managed-by: helm
name: dns-operator-manager-role
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
Expand Down
8 changes: 7 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ func main() {
var providers stringSliceFlags
var dnsProbesEnabled bool
var allowInsecureCerts bool
var configmapTransport bool

flag.BoolVar(&configmapTransport, "e2e-test-only-use-mock-health-transport", false, "This enables a mock transport for health checks based on a configmap in the dns-operator namespace named test-health-checks. It will stop proper health checks happening")

flag.BoolVar(&dnsProbesEnabled, "enable-probes", true, "Enable DNSHealthProbes controller.")
flag.BoolVar(&allowInsecureCerts, "insecure-health-checks", true, "Allow DNSHealthProbes to use insecure certificates")
Expand Down Expand Up @@ -161,7 +164,10 @@ func main() {
}

if dnsProbesEnabled {
probeManager := probes.NewProbeManager()
probeManager := probes.NewProbeManager(configmapTransport)
if configmapTransport {
setupLog.Info("WARNING configmap based transport enabled for health checks")
}
if err = (&controller.DNSProbeReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Expand Down
17 changes: 17 additions & 0 deletions config/deploy/mock-transport/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Local deployment overlay.
#
# Set mock transport layer arg for deployment
#

resources:
- ../local

patches:
- patch: |-
- op: add
path: /spec/template/spec/containers/0/args/-
value: --e2e-test-only-use-mock-health-transport
target:
kind: Deployment
name: controller-manager
namespace: system
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ secretGenerator:
- name: dns-provider-credentials
type: "kuadrant.io/inmemory"
literals:
- INMEM_INIT_ZONES=kuadrant.local
- INMEM_INIT_ZONES=e2e.hcpapps.net
12 changes: 12 additions & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ kind: ClusterRole
metadata:
name: manager-role
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
Expand Down
1 change: 1 addition & 0 deletions internal/controller/dnshealthcheckprobe_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ type DNSProbeReconciler struct {
//+kubebuilder:rbac:groups=kuadrant.io,resources=dnshealthcheckprobes,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=kuadrant.io,resources=dnshealthcheckprobes/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=kuadrant.io,resources=dnshealthcheckprobes/finalizers,verbs=update
//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete

func (r *DNSProbeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
baseLogger := log.FromContext(ctx).WithName("dnsprobe_controller")
Expand Down
8 changes: 8 additions & 0 deletions internal/controller/dnsrecord_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ import (
"github.com/kuadrant/dns-operator/internal/provider"
)

type HealthCheckOption bool

const (
DNSRecordFinalizer = "kuadrant.io/dns-record"
validationRequeueVariance = 0.5
Expand All @@ -58,6 +60,11 @@ const (
txtRegistryEncryptEnabled = false
txtRegistryEncryptAESKey = ""
txtRegistryCacheInterval = time.Duration(0)

EnableHealthCheckProbes HealthCheckOption = true
DisableHealthCheckProbes HealthCheckOption = false
EnableHealthCheckInsecureEndpoints HealthCheckOption = true
DisableHealthCheckInsecureEndpoints HealthCheckOption = false
)

var (
Expand Down Expand Up @@ -446,6 +453,7 @@ func setStatusConditions(record *v1alpha1.DNSRecord, hadChanges bool, notHealthy

// probes are disabled or not defined
if record.Spec.HealthCheck == nil {
meta.RemoveStatusCondition(&record.Status.Conditions, string(v1alpha1.ConditionTypeHealthy))
return
}

Expand Down
2 changes: 1 addition & 1 deletion internal/controller/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ var _ = BeforeSuite(func() {
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ProviderFactory: providerFactory,
}).SetupWithManager(mgr, RequeueDuration, ValidityDuration, DefaultValidationDuration, true, true)
}).SetupWithManager(mgr, RequeueDuration, ValidityDuration, DefaultValidationDuration, bool(EnableHealthCheckProbes), bool(EnableHealthCheckInsecureEndpoints))
Expect(err).ToNot(HaveOccurred())

go func() {
Expand Down
47 changes: 45 additions & 2 deletions internal/probes/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import (
"fmt"
"net"
"net/http"
"strconv"
"time"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilnet "k8s.io/apimachinery/pkg/util/net"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -203,11 +205,14 @@ func TransportWithDNSResponse(overrides map[string]string, allowInsecureCertific

type ProbeManager struct {
probes map[string]context.CancelFunc
//main use for this is to configure a test transport
overrideTransport bool
}

func NewProbeManager() *ProbeManager {
func NewProbeManager(overrideTransport bool) *ProbeManager {
return &ProbeManager{
probes: map[string]context.CancelFunc{},
probes: map[string]context.CancelFunc{},
overrideTransport: overrideTransport,
}
}

Expand Down Expand Up @@ -285,9 +290,47 @@ func (m *ProbeManager) EnsureProbeWorker(ctx context.Context, k8sClient client.C
// Either worker does not exist, or gen changed and old worker got killed. Creating a new one.
logger.V(2).Info("health: starting fresh worker for", "generation", probeCR.Generation, "probe", keyForProbe(probeCR))
probe := NewProbe(headers)
if m.overrideTransport {
rt, err := m.EnableConfigmapTransport(ctx, k8sClient, probeCR.Spec.Address)
if err != nil {
logger.Error(err, "failed to setup configmap transport")
return
}
probe.Transport = rt
}
m.probes[keyForProbe(probeCR)] = probe.Start(ctx, k8sClient, probeCR)
}

func keyForProbe(probe *v1alpha1.DNSHealthCheckProbe) string {
return fmt.Sprintf("%s/%s", probe.Name, probe.Namespace)
}

func (m *ProbeManager) EnableConfigmapTransport(ctx context.Context, k8sClient client.Client, ip string) (RoundTripperFunc, error) {
//debug and testing configmap
rt := func(r *http.Request) (*http.Response, error) {
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "mockhealthcheckresponses",
// we cannot guarantee this ns to be present
Namespace: "dns-operator-system",
},
}
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
return nil, fmt.Errorf("failed to load test only configmap based transport %w", err)
}
fmt.Println("CALLED MOCK HELATH CHECK")
if reponseCode, ok := cm.Data[ip]; ok {
code, err := strconv.ParseInt(reponseCode, 10, 64)
if err != nil {
fmt.Println("failed to parse response code from configmap", err)
return nil, fmt.Errorf("failed to parse response code from configmap %w", err)
}
return &http.Response{
StatusCode: int(code),
}, nil
}
fmt.Println("no response code in configmap for uri", r.URL.Host)
return nil, fmt.Errorf("no response code in configmap for uri %s", r.URL.Host)
}
return rt, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,19 @@ metadata:
name: ${testID}
namespace: ${testNamespace}
spec:
healthCheck:
port: 80
path: "/health"
protocol: "HTTP"
interval: 1s
failureThreshold: 5
endpoints:
- dnsName: 14byhk-2k52h1.klb.${testHostname}
recordTTL: 60
recordType: A
targets:
- 172.32.200.1
- 172.32.200.2
- dnsName: ${testHostname}
recordTTL: 300
recordType: CNAME
Expand Down
48 changes: 48 additions & 0 deletions test/e2e/fixtures/healthcheck_test/geo-dnsrecord.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
apiVersion: kuadrant.io/v1alpha1
kind: DNSRecord
metadata:
name: ${testID}
namespace: ${testNamespace}
spec:
endpoints:
- dnsName: 14byhk-2k52h1.klb.${testHostname}
recordTTL: 60
recordType: A
targets:
- 172.32.200.1
- dnsName: ${testHostname}
recordTTL: 300
recordType: CNAME
targets:
- klb.${testHostname}
- dnsName: eu.klb.${testHostname}
providerSpecific:
- name: weight
value: "120"
recordTTL: 60
recordType: CNAME
setIdentifier: 14byhk-2k52h1.klb.${testHostname}
targets:
- 14byhk-2k52h1.klb.${testHostname}
- dnsName: klb.${testHostname}
providerSpecific:
- name: geo-code
value: ${testGeoCode}
recordTTL: 300
recordType: CNAME
setIdentifier: ${testGeoCode}
targets:
- eu.klb.${testHostname}
- dnsName: klb.${testHostname}
providerSpecific:
- name: geo-code
value: '*'
recordTTL: 300
recordType: CNAME
setIdentifier: default
targets:
- eu.klb.${testHostname}
providerRef:
name: ${TEST_DNS_PROVIDER_SECRET_NAME}
ownerID: 2bq03i
rootHost: ${testHostname}
Loading

0 comments on commit 8cc4593

Please sign in to comment.