diff --git a/.gitignore b/.gitignore index 9bb3a9bec8..dbaa94f982 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,8 @@ changes cosign.key *.sig *.att + +# Terraform +terraform.tfstate* +.terraform.lock.hcl +.terraform/ diff --git a/Makefile b/Makefile index 97b3ece241..b8db43285f 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ CLUSTER_IDS = $(shell seq $(CLUSTERS_NUMBER)) CLUSTER_NAME ?= test-gslb CLUSTER_GEO_TAGS ?= eu us cz af ru ap uk ca CHART ?= k8gb/k8gb +CHART_TEST ?= ./chart/k8gb CLUSTER_GSLB_NETWORK = k3d-action-bridge-network CLUSTER_GSLB_GATEWAY = docker network inspect ${CLUSTER_GSLB_NETWORK} -f '{{ (index .IPAM.Config 0).Gateway }}' GSLB_DOMAIN ?= cloud.example.com @@ -136,7 +137,7 @@ deploy-full-local-setup: ensure-cluster-size ## Deploy full local multicluster s $(MAKE) create-local-cluster CLUSTER_NAME=$(CLUSTER_NAME)$$c ;\ done @if [ "$(K8GB_LOCAL_VERSION)" = test ]; then $(MAKE) release-images ; fi - $(MAKE) deploy-$(K8GB_LOCAL_VERSION)-version DEPLOY_APPS=true + $(MAKE) deploy-$(K8GB_LOCAL_VERSION)-version DEPLOY_APPS=false .PHONY: deploy-stable-version deploy-stable-version: @@ -156,7 +157,7 @@ deploy-test-version: ## Upgrade k8gb to the test version on existing clusters done @for c in $(CLUSTER_IDS); do \ - $(MAKE) deploy-local-cluster CLUSTER_ID=$$c VERSION=$(SEMVER)-$(ARCH) CHART='./chart/k8gb' ;\ + $(MAKE) deploy-local-cluster CLUSTER_ID=$$c VERSION=$(SEMVER)-$(ARCH) CHART="$(CHART_TEST)" ;\ kubectl apply -n k8gb -f ./deploy/test/coredns-tcp-svc.yaml ;\ done diff --git a/dns-providers/README.md b/dns-providers/README.md new file mode 100644 index 0000000000..1bbfad66c6 --- /dev/null +++ b/dns-providers/README.md @@ -0,0 +1,3 @@ +# DNS provider tests + +In this space we document setups where we connect a local cluster to upstream DNS providers with the goal of testing the integrations. diff --git a/dns-providers/azure/README.md b/dns-providers/azure/README.md new file mode 100644 index 0000000000..4abcde784d --- /dev/null +++ b/dns-providers/azure/README.md @@ -0,0 +1,114 @@ +# Test Azure DNS integration from a local cluster + +This is a guide how to test the Azure DNS integration of K8GB + +## Azure infrastructure + +### Azure subscription + +First you will need an Azure subscription, if you don't have one already you can get started with a [free account](https://azure.microsoft.com/en-us/pricing/purchase-options/azure-account). +Afterwards, login to your subscription in your terminal using `az login`. + +Afterwards store your Azure subcription ID in an environment variable. We will use it as a terraform variable. +``` +export ARM_SUBSCRIPTION_ID="$(az account show --query id -o tsv)" +``` + +### DNS Zone and service principal + +The next step is to create a DNS zone and a service principal that allows K8GB to modify records in the zone. +You can use the terraform code provided in the `terraform` folder to get started. You will be prompted with the name of the DNS zone. The name needs to be unique in Azure, but you don't need to own the zone for the purpose of this guide: +``` +$ cd terraform +$ terraform init +$ terraform apply +var.dns_zone_name + Name of the DNS zone + + Enter a value: k8gb.io +``` + +You will now have a service principal. You can retrieve its ID and secret with the following commands +``` +terraform output service_principal_client_id +terraform output service_principal_client_secret +``` + +### Create local clusters + +We have everything we need from Azure, we can now create a local cluster. +This command will create the clusters `k3d-test-gslb1` and `k3d-test-gslb2`, and install k8gb from the branch you are on: +``` +K8GB_LOCAL_VERSION=test FULL_LOCAL_SETUP_WITH_APPS=false make deploy-full-local-setup +``` + +### Connect K8GB to Azure + +At this moment K8GB is using the upstream DNS server running on the local cluster `k3d-edgedns`. We want to point it to the DNS infrastructure we created in Azure. + +To do that we will need to create a secret on both clusters, on the `k8gb` namespace with the name `external-dns-secret-azure`. The format of the secret is documented in the [external dns docs](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/azure.md#creating-a-configuration-file-for-the-service-principal). If you are in your `terraform` folder you can create it using: +``` +cat <<-EOF > azure.json +{ + "tenantId": "$(az account show --query tenantId -o tsv)", + "subscriptionId": "$(az account show --query id -o tsv)", + "resourceGroup": "rg-k8gb", + "aadClientId": "$(terraform output --raw service_principal_client_id)", + "aadClientSecret": "$(terraform output --raw service_principal_client_secret)" +} +EOF +``` +Now apply the secret to both of the clusters: +``` +kubectl create secret generic external-dns-secret-azure -n k8gb --from-file azure.json --context k3d-test-gslb1 +kubectl create secret generic external-dns-secret-azure -n k8gb --from-file azure.json --context k3d-test-gslb2 +``` + +### Create application + +Finally, we can create a GSLB resouce that will trigger a reconciliation loop of the controller and configure DNS name delegation on Azure. +To do that we will need to configure the DNS zone we create on K8GB: +``` +# replace with your zone +EDGE_DNS_ZONE="k8gb.io" +``` +``` +DNS_ZONE="cloud.${EDGE_DNS_ZONE}" +EDGE_DNS_SERVER=$(az network dns record-set ns list --resource-group rg-k8gb --zone-name "$EDGE_DNS_ZONE" --query "[?name=='@'].NSRecords[0].nsdname" --output tsv | sed 's/\.$//') +``` + +``` +cd helm +helm package -u . > /dev/null && helm template k8gb k8gb-v0.1.0.tgz -n k8gb -f values.yaml -f values-eu.yaml --set "k8gb.k8gb.dnsZone=$DNS_ZONE" --set "k8gb.k8gb.edgeDNSZone=$EDGE_DNS_ZONE" --set "k8gb.k8gb.edgeDNSServers[0]=$EDGE_DNS_SERVER" > manifests-eu.yaml +helm package -u . > /dev/null && helm template k8gb k8gb-v0.1.0.tgz -n k8gb -f values.yaml -f values-us.yaml --set "k8gb.k8gb.dnsZone=$DNS_ZONE" --set "k8gb.k8gb.edgeDNSZone=$EDGE_DNS_ZONE" --set "k8gb.k8gb.edgeDNSServers[0]=$EDGE_DNS_SERVER" > manifests-us.yaml + +kubectl apply -f manifests-eu.yaml --context k3d-test-gslb1 +kubectl apply -f manifests-us.yaml --context k3d-test-gslb2 +``` + +### Verify zone delegation in Azure + +And voila, our local clusters are now integrated with Azure. We can quickly verify everything is working. + +In Azure we should find the following records (the IP addresses may be different depending on the allocation by docker): +| Name | Type | Value | +| -------- | ------- | ------- | +| cloud | NS | gslb-ns-eu-cloud.k8gb.io gslb-ns-us-cloud.k8gb.io +| gslb-ns-eu-cloud | A | 172.18.0.6 172.18.0.7 +| gslb-ns-us-cloud | A | 172.18.0.10 172.18.0.11 +``` +az network dns record-set a list --resource-group rg-k8gb --zone-name "$EDGE_DNS_ZONE" --output json +az network dns record-set ns list --resource-group rg-k8gb --zone-name "$EDGE_DNS_ZONE" --output json +``` + +You can also fetch the records using the following DNS query: +``` +dig @${EDGE_DNS_SERVER} cloud.k8gb.io +... +;; AUTHORITY SECTION: +cloud.k8gb.io. 5 IN NS gslb-ns-eu-cloud.k8gb.io. +cloud.k8gb.io. 5 IN NS gslb-ns-us-cloud.k8gb.io. +... +``` + +Unfortunately the A records cannot be queried because they are private IP addresses and Azure does not return them in a public DNS zone, but this is enough for testing. diff --git a/dns-providers/azure/helm/Chart.yaml b/dns-providers/azure/helm/Chart.yaml new file mode 100644 index 0000000000..b25620eb09 --- /dev/null +++ b/dns-providers/azure/helm/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: k8gb +description: A Helm chart for Kubernetes Global Balancer +icon: https://www.k8gb.io/assets/images/icon-192x192.png +type: application +version: v0.1.0 +dependencies: + - name: k8gb + repository: file://../../../chart/k8gb + version: v0.14.0 + +home: https://www.k8gb.io/ +sources: + - https://github.com/k8gb-io/k8gb diff --git a/dns-providers/azure/helm/templates/gslb.yaml b/dns-providers/azure/helm/templates/gslb.yaml new file mode 100644 index 0000000000..742bcd6c21 --- /dev/null +++ b/dns-providers/azure/helm/templates/gslb.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: failover-playground-istio + namespace: test-azure +spec: + resourceRef: + apiVersion: networking.istio.io/v1 + kind: VirtualService + matchLabels: + app: failover-playground-istio + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + name: failover-playground-istio + namespace: test-azure + labels: + app: failover-playground-istio +spec: + gateways: + - istio-ingress/failover-playground-istio + hosts: + - failover-playground-istio.{{ .Values.k8gb.k8gb.dnsZone }} + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 +--- +apiVersion: networking.istio.io/v1 +kind: Gateway +metadata: + name: failover-playground-istio + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - failover-playground-istio.{{ .Values.k8gb.k8gb.dnsZone }} + port: + name: http + number: 8080 + protocol: http diff --git a/dns-providers/azure/helm/templates/ns.yaml b/dns-providers/azure/helm/templates/ns.yaml new file mode 100644 index 0000000000..5363abe449 --- /dev/null +++ b/dns-providers/azure/helm/templates/ns.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-azure + labels: + istio-injection: enabled diff --git a/dns-providers/azure/helm/templates/podinfo.yaml b/dns-providers/azure/helm/templates/podinfo.yaml new file mode 100644 index 0000000000..e69a7c91ce --- /dev/null +++ b/dns-providers/azure/helm/templates/podinfo.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-podinfo + namespace: test-azure + labels: + app.kubernetes.io/name: frontend-podinfo +spec: + type: ClusterIP + ports: + - port: 9898 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: frontend-podinfo +--- +apiVersion: v1 +kind: Pod +metadata: + name: frontend-podinfo + namespace: test-azure + labels: + app.kubernetes.io/name: frontend-podinfo +spec: + containers: + - name: podinfo + image: "ghcr.io/stefanprodan/podinfo:5.1.1" + command: + - ./podinfo + - --port=9898 + ports: + - name: http + containerPort: 9898 + protocol: TCP + resources: + requests: + memory: 64Mi + cpu: 250m + limits: + memory: 128Mi + cpu: 500m diff --git a/dns-providers/azure/helm/values-eu.yaml b/dns-providers/azure/helm/values-eu.yaml new file mode 100644 index 0000000000..19bb87ff52 --- /dev/null +++ b/dns-providers/azure/helm/values-eu.yaml @@ -0,0 +1,4 @@ +k8gb: + k8gb: + clusterGeoTag: "eu" + extGslbClustersGeoTags: "us" diff --git a/dns-providers/azure/helm/values-us.yaml b/dns-providers/azure/helm/values-us.yaml new file mode 100644 index 0000000000..09c30e2b07 --- /dev/null +++ b/dns-providers/azure/helm/values-us.yaml @@ -0,0 +1,4 @@ +k8gb: + k8gb: + clusterGeoTag: "us" + extGslbClustersGeoTags: "eu" diff --git a/dns-providers/azure/helm/values.yaml b/dns-providers/azure/helm/values.yaml new file mode 100644 index 0000000000..52b6849f2b --- /dev/null +++ b/dns-providers/azure/helm/values.yaml @@ -0,0 +1,12 @@ +k8gb: + k8gb: + dnsZone: "" + edgeDNSZone: "" + edgeDNSServers: + - "