From daa0523b090b1722dd64cfc111dce230249e8822 Mon Sep 17 00:00:00 2001 From: Andre Aguas Date: Sat, 2 Nov 2024 14:16:56 +0100 Subject: [PATCH] Create test setup for Azure DNS integration As stated in #1772, we would like to have test setup with the main DNS providers. This PR creates a complete guide documenting how to test the Azure integration, including terraform and helm configuration. According to the [docs](https://azure.microsoft.com/en-us/pricing/details/dns/) the infrastructure should cost $0.90 per month, but it can also be destroyed and re-provisioned anytime: ``` First 25 hosted DNS zones $0.50 per zone per month First billion DNS queries/month $0.40 per million ``` Signed-off-by: Andre Aguas --- .gitignore | 5 + dns-providers/README.md | 3 + dns-providers/azure/README.md | 114 ++++++++++++++++++ dns-providers/azure/helm/Chart.yaml | 14 +++ dns-providers/azure/helm/templates/gslb.yaml | 51 ++++++++ dns-providers/azure/helm/templates/ns.yaml | 6 + .../azure/helm/templates/podinfo.yaml | 43 +++++++ dns-providers/azure/helm/values-eu.yaml | 4 + dns-providers/azure/helm/values-us.yaml | 4 + dns-providers/azure/helm/values.yaml | 12 ++ dns-providers/azure/terraform/main.tf | 38 ++++++ dns-providers/azure/terraform/output.tf | 10 ++ dns-providers/azure/terraform/providers.tf | 17 +++ dns-providers/azure/terraform/variables.tf | 16 +++ 14 files changed, 337 insertions(+) create mode 100644 dns-providers/README.md create mode 100644 dns-providers/azure/README.md create mode 100644 dns-providers/azure/helm/Chart.yaml create mode 100644 dns-providers/azure/helm/templates/gslb.yaml create mode 100644 dns-providers/azure/helm/templates/ns.yaml create mode 100644 dns-providers/azure/helm/templates/podinfo.yaml create mode 100644 dns-providers/azure/helm/values-eu.yaml create mode 100644 dns-providers/azure/helm/values-us.yaml create mode 100644 dns-providers/azure/helm/values.yaml create mode 100644 dns-providers/azure/terraform/main.tf create mode 100644 dns-providers/azure/terraform/output.tf create mode 100644 dns-providers/azure/terraform/providers.tf create mode 100644 dns-providers/azure/terraform/variables.tf diff --git a/.gitignore b/.gitignore index 9bb3a9bec8..dbaa94f982 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,8 @@ changes cosign.key *.sig *.att + +# Terraform +terraform.tfstate* +.terraform.lock.hcl +.terraform/ diff --git a/dns-providers/README.md b/dns-providers/README.md new file mode 100644 index 0000000000..1bbfad66c6 --- /dev/null +++ b/dns-providers/README.md @@ -0,0 +1,3 @@ +# DNS provider tests + +In this space we document setups where we connect a local cluster to upstream DNS providers with the goal of testing the integrations. diff --git a/dns-providers/azure/README.md b/dns-providers/azure/README.md new file mode 100644 index 0000000000..4abcde784d --- /dev/null +++ b/dns-providers/azure/README.md @@ -0,0 +1,114 @@ +# Test Azure DNS integration from a local cluster + +This is a guide how to test the Azure DNS integration of K8GB + +## Azure infrastructure + +### Azure subscription + +First you will need an Azure subscription, if you don't have one already you can get started with a [free account](https://azure.microsoft.com/en-us/pricing/purchase-options/azure-account). +Afterwards, login to your subscription in your terminal using `az login`. + +Afterwards store your Azure subcription ID in an environment variable. We will use it as a terraform variable. +``` +export ARM_SUBSCRIPTION_ID="$(az account show --query id -o tsv)" +``` + +### DNS Zone and service principal + +The next step is to create a DNS zone and a service principal that allows K8GB to modify records in the zone. +You can use the terraform code provided in the `terraform` folder to get started. You will be prompted with the name of the DNS zone. The name needs to be unique in Azure, but you don't need to own the zone for the purpose of this guide: +``` +$ cd terraform +$ terraform init +$ terraform apply +var.dns_zone_name + Name of the DNS zone + + Enter a value: k8gb.io +``` + +You will now have a service principal. You can retrieve its ID and secret with the following commands +``` +terraform output service_principal_client_id +terraform output service_principal_client_secret +``` + +### Create local clusters + +We have everything we need from Azure, we can now create a local cluster. +This command will create the clusters `k3d-test-gslb1` and `k3d-test-gslb2`, and install k8gb from the branch you are on: +``` +K8GB_LOCAL_VERSION=test FULL_LOCAL_SETUP_WITH_APPS=false make deploy-full-local-setup +``` + +### Connect K8GB to Azure + +At this moment K8GB is using the upstream DNS server running on the local cluster `k3d-edgedns`. We want to point it to the DNS infrastructure we created in Azure. + +To do that we will need to create a secret on both clusters, on the `k8gb` namespace with the name `external-dns-secret-azure`. The format of the secret is documented in the [external dns docs](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/azure.md#creating-a-configuration-file-for-the-service-principal). If you are in your `terraform` folder you can create it using: +``` +cat <<-EOF > azure.json +{ + "tenantId": "$(az account show --query tenantId -o tsv)", + "subscriptionId": "$(az account show --query id -o tsv)", + "resourceGroup": "rg-k8gb", + "aadClientId": "$(terraform output --raw service_principal_client_id)", + "aadClientSecret": "$(terraform output --raw service_principal_client_secret)" +} +EOF +``` +Now apply the secret to both of the clusters: +``` +kubectl create secret generic external-dns-secret-azure -n k8gb --from-file azure.json --context k3d-test-gslb1 +kubectl create secret generic external-dns-secret-azure -n k8gb --from-file azure.json --context k3d-test-gslb2 +``` + +### Create application + +Finally, we can create a GSLB resouce that will trigger a reconciliation loop of the controller and configure DNS name delegation on Azure. +To do that we will need to configure the DNS zone we create on K8GB: +``` +# replace with your zone +EDGE_DNS_ZONE="k8gb.io" +``` +``` +DNS_ZONE="cloud.${EDGE_DNS_ZONE}" +EDGE_DNS_SERVER=$(az network dns record-set ns list --resource-group rg-k8gb --zone-name "$EDGE_DNS_ZONE" --query "[?name=='@'].NSRecords[0].nsdname" --output tsv | sed 's/\.$//') +``` + +``` +cd helm +helm package -u . > /dev/null && helm template k8gb k8gb-v0.1.0.tgz -n k8gb -f values.yaml -f values-eu.yaml --set "k8gb.k8gb.dnsZone=$DNS_ZONE" --set "k8gb.k8gb.edgeDNSZone=$EDGE_DNS_ZONE" --set "k8gb.k8gb.edgeDNSServers[0]=$EDGE_DNS_SERVER" > manifests-eu.yaml +helm package -u . > /dev/null && helm template k8gb k8gb-v0.1.0.tgz -n k8gb -f values.yaml -f values-us.yaml --set "k8gb.k8gb.dnsZone=$DNS_ZONE" --set "k8gb.k8gb.edgeDNSZone=$EDGE_DNS_ZONE" --set "k8gb.k8gb.edgeDNSServers[0]=$EDGE_DNS_SERVER" > manifests-us.yaml + +kubectl apply -f manifests-eu.yaml --context k3d-test-gslb1 +kubectl apply -f manifests-us.yaml --context k3d-test-gslb2 +``` + +### Verify zone delegation in Azure + +And voila, our local clusters are now integrated with Azure. We can quickly verify everything is working. + +In Azure we should find the following records (the IP addresses may be different depending on the allocation by docker): +| Name | Type | Value | +| -------- | ------- | ------- | +| cloud | NS | gslb-ns-eu-cloud.k8gb.io gslb-ns-us-cloud.k8gb.io +| gslb-ns-eu-cloud | A | 172.18.0.6 172.18.0.7 +| gslb-ns-us-cloud | A | 172.18.0.10 172.18.0.11 +``` +az network dns record-set a list --resource-group rg-k8gb --zone-name "$EDGE_DNS_ZONE" --output json +az network dns record-set ns list --resource-group rg-k8gb --zone-name "$EDGE_DNS_ZONE" --output json +``` + +You can also fetch the records using the following DNS query: +``` +dig @${EDGE_DNS_SERVER} cloud.k8gb.io +... +;; AUTHORITY SECTION: +cloud.k8gb.io. 5 IN NS gslb-ns-eu-cloud.k8gb.io. +cloud.k8gb.io. 5 IN NS gslb-ns-us-cloud.k8gb.io. +... +``` + +Unfortunately the A records cannot be queried because they are private IP addresses and Azure does not return them in a public DNS zone, but this is enough for testing. diff --git a/dns-providers/azure/helm/Chart.yaml b/dns-providers/azure/helm/Chart.yaml new file mode 100644 index 0000000000..b25620eb09 --- /dev/null +++ b/dns-providers/azure/helm/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: k8gb +description: A Helm chart for Kubernetes Global Balancer +icon: https://www.k8gb.io/assets/images/icon-192x192.png +type: application +version: v0.1.0 +dependencies: + - name: k8gb + repository: file://../../../chart/k8gb + version: v0.14.0 + +home: https://www.k8gb.io/ +sources: + - https://github.com/k8gb-io/k8gb diff --git a/dns-providers/azure/helm/templates/gslb.yaml b/dns-providers/azure/helm/templates/gslb.yaml new file mode 100644 index 0000000000..742bcd6c21 --- /dev/null +++ b/dns-providers/azure/helm/templates/gslb.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: failover-playground-istio + namespace: test-azure +spec: + resourceRef: + apiVersion: networking.istio.io/v1 + kind: VirtualService + matchLabels: + app: failover-playground-istio + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + name: failover-playground-istio + namespace: test-azure + labels: + app: failover-playground-istio +spec: + gateways: + - istio-ingress/failover-playground-istio + hosts: + - failover-playground-istio.{{ .Values.k8gb.k8gb.dnsZone }} + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 +--- +apiVersion: networking.istio.io/v1 +kind: Gateway +metadata: + name: failover-playground-istio + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - failover-playground-istio.{{ .Values.k8gb.k8gb.dnsZone }} + port: + name: http + number: 8080 + protocol: http diff --git a/dns-providers/azure/helm/templates/ns.yaml b/dns-providers/azure/helm/templates/ns.yaml new file mode 100644 index 0000000000..5363abe449 --- /dev/null +++ b/dns-providers/azure/helm/templates/ns.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-azure + labels: + istio-injection: enabled diff --git a/dns-providers/azure/helm/templates/podinfo.yaml b/dns-providers/azure/helm/templates/podinfo.yaml new file mode 100644 index 0000000000..e69a7c91ce --- /dev/null +++ b/dns-providers/azure/helm/templates/podinfo.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-podinfo + namespace: test-azure + labels: + app.kubernetes.io/name: frontend-podinfo +spec: + type: ClusterIP + ports: + - port: 9898 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: frontend-podinfo +--- +apiVersion: v1 +kind: Pod +metadata: + name: frontend-podinfo + namespace: test-azure + labels: + app.kubernetes.io/name: frontend-podinfo +spec: + containers: + - name: podinfo + image: "ghcr.io/stefanprodan/podinfo:5.1.1" + command: + - ./podinfo + - --port=9898 + ports: + - name: http + containerPort: 9898 + protocol: TCP + resources: + requests: + memory: 64Mi + cpu: 250m + limits: + memory: 128Mi + cpu: 500m diff --git a/dns-providers/azure/helm/values-eu.yaml b/dns-providers/azure/helm/values-eu.yaml new file mode 100644 index 0000000000..19bb87ff52 --- /dev/null +++ b/dns-providers/azure/helm/values-eu.yaml @@ -0,0 +1,4 @@ +k8gb: + k8gb: + clusterGeoTag: "eu" + extGslbClustersGeoTags: "us" diff --git a/dns-providers/azure/helm/values-us.yaml b/dns-providers/azure/helm/values-us.yaml new file mode 100644 index 0000000000..09c30e2b07 --- /dev/null +++ b/dns-providers/azure/helm/values-us.yaml @@ -0,0 +1,4 @@ +k8gb: + k8gb: + clusterGeoTag: "us" + extGslbClustersGeoTags: "eu" diff --git a/dns-providers/azure/helm/values.yaml b/dns-providers/azure/helm/values.yaml new file mode 100644 index 0000000000..52b6849f2b --- /dev/null +++ b/dns-providers/azure/helm/values.yaml @@ -0,0 +1,12 @@ +k8gb: + k8gb: + dnsZone: "" + edgeDNSZone: "" + edgeDNSServers: + - "