diff --git a/.dockerignore b/.dockerignore index 039ef46..ca3b34f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .github/ ci/ +kind/ .release-please-manifest.json release-please-manifest.json README.md diff --git a/.gitignore b/.gitignore index bce9047..d5ba8c1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ go.work go.work.sum tmp/ +__pycache__ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d33f4ef..6bed458 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # Dev stage FROM --platform=${BUILDPLATFORM:-linux/amd64} docker.io/golang:1.22.4 AS dev + RUN go install github.com/air-verse/air@latest WORKDIR /go/src/app COPY go.mod go.mod diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..dde1fa2 --- /dev/null +++ b/Makefile @@ -0,0 +1,117 @@ +SHELL := /bin/bash +DOCKER := $(shell type -p docker) +KIND := $(shell type -p kind) +HELM := $(shell type -p helm) +KUBECTL := $(shell type -p helm) + +WEBHOOK_IMAGE := external-dns-midaas-webhook:dev +WEBHOOK_FOLDER := ./ + +MIDAAS_IMAGE := api-midaas:dev +MIDAAS_FOLDER := ./contribute/midaas-ws/ + +export MIDAAS_WS_URL ?= http://midaas.default:8080/ws/ +export MIDAAS_DEV_SUFFIX ?= dev.local +export MIDAAS_ENV_KEYNAME ?= d1 +export MIDAAS_ENV_KEYVALUE ?= test +export MIDAAS_ENV_ZONES ?= $(MIDAAS_ENV_KEYNAME).$(MIDAAS_DEV_SUFFIX) + +KIND_CLUSTER_NAME ?= midaas +KIND_INGRESS_CONTROLLER = NGINX + +# Commons targets + +all: deploy-MIDAAS deploy-WEBHOOK create-test-ingress + +clean: delete-cluster delete-image-MIDAAS delete-image-WEBHOOK + +create-test-ingress: create-cluster check-prerequisites-kubectl + @kubectl apply -f ./contribute/ressources/ingress.yaml + +logs-%: + @kubectl logs -f deployments/external-dns -c $* + +midaas-get-zone: + @kubectl exec midaas cat /tmp/$(MIDAAS_ENV_KEYNAME).$(MIDAAS_DEV_SUFFIX) | jq + +# Check prerequisites + +check-prerequisites-docker: +ifeq ("$(wildcard ${DOCKER})","") + @echo "docker not found" ; exit 1 +endif +check-prerequisites-kind: +ifeq ("$(wildcard ${KIND})","") + @echo "'kind' not found" ; exit 1 +endif +check-prerequisites-kubectl: +ifeq ("$(wildcard ${KUBECTL})","") + @echo "'kubectl' not found" ; exit 1 +endif +check-prerequisites-helm: +ifeq ("$(wildcard ${HELM})","") + @echo "'helm' not found" ; exit 1 +endif + +# Kind targets + +create-cluster: check-prerequisites-kind +ifeq ($(shell kind get clusters |grep $(KIND_CLUSTER_NAME)), $(KIND_CLUSTER_NAME)) + @echo "Kind cluster '$(KIND_CLUSTER_NAME)' already exists, skipping" +else + @kind create cluster --name $(KIND_CLUSTER_NAME) --config ./contribute/kind/kind-config.yaml + @kubectl config use-context kind-$(KIND_CLUSTER_NAME) +endif + +delete-cluster: check-prerequisites-kind +ifeq ($(shell kind get clusters |grep $(KIND_CLUSTER_NAME)), $(KIND_CLUSTER_NAME)) + @kind delete cluster --name $(KIND_CLUSTER_NAME) +endif + +start-ingress-controller: create-cluster +ifeq ($(KIND_INGRESS_CONTROLLER), NGINX) + @if [ ! -s /tmp/external-dns-nginx.yaml ]; then curl -Ls https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml > /tmp/external-dns-nginx.yaml; fi + @kubectl apply -f /tmp/external-dns-nginx.yaml + @kubectl wait --namespace ingress-nginx \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/component=controller \ + --timeout=90s +else ifeq ($(KIND_INGRESS_CONTROLLER), TRAEFIK) + @echo traefik +endif + +# Docker build and push targets +build-%: check-prerequisites-docker + @if [ $* = WEBHOOK ]; then docker build --target dev ${OPTIONS} -t ${$*_IMAGE} ${$*_FOLDER}; else docker build ${OPTIONS} -t ${$*_IMAGE} ${$*_FOLDER}; fi + +push-%: check-prerequisites-kind + kind load docker-image ${$*_IMAGE} --name $(KIND_CLUSTER_NAME) + +delete-image-%: check-prerequisites-docker + docker rmi ${$*_IMAGE} + +# Midaas deployment targets +delete-MIDAAS: create-cluster check-prerequisites-kubectl + @kubectl delete pod midaas --ignore-not-found + @kubectl delete svc midaas --ignore-not-found + +deploy-MIDAAS: create-cluster check-prerequisites-kubectl build-MIDAAS push-MIDAAS delete-MIDAAS + @kubectl run --image $(MIDAAS_IMAGE) --expose=true --port 8080 \ + --env "MIDAAS_KEYNAME=$(MIDAAS_ENV_KEYNAME)" \ + --env "MIDAAS_KEYVALUE=$(MIDAAS_ENV_KEYVALUE)" \ + --env "MIDAAS_ZONES=$(MIDAAS_ENV_ZONES)" midaas + @echo "Kubernetes midaas service is listening on port 8080" + + +# Webhook deployment targets + +delete-WEBHOOK: create-cluster check-prerequisites-helm + @if [ "external-dns" == "$(shell helm ls -f external-dns -o json |jq -r .[].name)" ]; then helm delete external-dns; else echo "No external-dns release is currently running"; fi + +deploy-WEBHOOK: start-ingress-controller check-prerequisites-helm build-WEBHOOK push-WEBHOOK delete-WEBHOOK + @echo "Adding repository" + @helm repo add external-dns https://kubernetes-sigs.github.io/external-dns/ + @envsubst < ./contribute/ressources/external-dns-values.yaml > /tmp/external-dns-values.yaml + @helm upgrade --force --install external-dns external-dns/external-dns -f /tmp/external-dns-values.yaml + @echo "external DNS is running with webhook in sidecar" + diff --git a/README.md b/README.md index 2aa253a..fbbe582 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ helm repo add external-dns https://kubernetes-sigs.github.io/external-dns/ 1. Create the helm values file `external-dns-midaas-values.yaml`: ```yaml +sources: + - ingress # -- How DNS records are synchronized between sources and providers; available values are `sync` & `upsert-only`. policy: sync # -- Specify the registry for storing ownership and labels. @@ -24,12 +26,13 @@ policy: sync # If `noop` midaas manage all records on zone registry: txt # can restrict zone -domainFilters: ["subzone.d1.dev.example.com"] +domainFilters: [] provider: name: webhook webhook: - image: ghcr.io/titigmr/external-dns-midaas-webhook - tag: latest + image: + repository: ghcr.io/titigmr/external-dns-midaas-webhook + tag: latest env: - name: PROVIDER_DNS_ZONE_SUFFIX value: "dev.example.com" @@ -42,7 +45,7 @@ provider: 2. Create helm deployment: ```sh -helm install external-dns external-dns -f external-dns-midaas-values.yaml +helm install external-dns external-dns/external-dns -f external-dns-midaas-values.yaml ``` ## Parameters references @@ -64,7 +67,62 @@ For example, `TSIG_ZONE_d1` with `PROVIDER_DNS_ZONE_SUFFIX` with `dev.example.co ## Local development -🚧 Work in progress. +### Prerequisite + +Download and install on your local machine: +- `make` in Debian/Ubuntu distrib with +```bash + sudo apt install build-essential +``` +- [docker](https://docs.docker.com/engine/install/) +- [kubectl](https://github.com/kubernetes/kubectl) +- [kind](https://github.com/kubernetes-sigs/kind) +- [helm](https://github.com/helm/helm) + +### Usage + + +You can create a development stack locally with this command: + +```sh +make +``` + +This target do the following target successively: +- `create-cluster` : create a `kind` cluster locally with an ingress controller configured +- `deploy-MIDAAS` : build, push and deploy `midaas` [webservice mock](./contribute/midaas-ws/) in the cluster +- `deploy-WEBHOOK` : build, push and deploy `external-dns` with the midaas webhook in development mode. You can modify the code with hot reload. + +For example, for restarting the webhook: + +```bash +make deploy-WEBHOOK +``` + +Don't forget create an ingress for trigger `external-dns`, an [example](./contribute/ressources/ingress.yaml) can be created with: + +```bash +make create-test-ingress +``` + +You can read the containers logs with: + +```bash +make logs-webhook +``` + +or + +```bash +make logs-external-dns +``` + +To clean all the components + +```sh +make clean +``` + ## Contributions diff --git a/contribute/README.md b/contribute/README.md new file mode 100644 index 0000000..81f894b --- /dev/null +++ b/contribute/README.md @@ -0,0 +1,38 @@ +# Run all stack locally with docker + +## Kind cluster + +One single node is deployed but it can be customized in `./kind/kind-config.yml`. The cluster comes with [Traefik](https://doc.traefik.io/traefik/providers/kubernetes-ingress/) or [Nginx](https://kind.sigs.k8s.io/docs/user/ingress/#ingress-nginx) ingress controller installed with port mapping on both ports `8080` and `8443`. + +The node is using `extraMounts` to provide a volume binding between host working directory and `/app` to give the ability to bind mount volumes into containers during development. + + +## Midaas Webservice + +A wrapper of midaas is available for development on folder `./midaas-ws`. Note that tool not really do dns records. It only writes fake domain on container filesystem. + +This webservice is written in python with `Fastapi` framework. The webservice listen on 3 endpoints: +- `GET` - `/ws/{domaine}` : retrieve all domains for a specific zone +- `PUT` - `/ws/{domaine}/{type}/{valeur}` : add or modify a DNS record +You must add this body in the request: +```json +{"ttl": 0, "keyname": "string", "keyvalue": "string"} +``` +- `DELETE` - `/ws/{domaine}/{type}/{valeur}` : add or modify a DNS +You must add this body in the request: +```json +{"keyname": "string", "keyvalue": "string"} +``` + +The midaas webservice can be configured with the following environment variables: + +| Name | Description | Default | +| --------------- | ---------------------- | --------- | +| MIDAAS_KEYNAME | TSIG Keyname | test | +| MIDAAS_KEYVALUE | TSIG Keyvalue | test | +| MIDAAS_ZONE | Zone managed by MiDaas | dev.local | + + +## External-DNS Locally + +:construction: \ No newline at end of file diff --git a/contribute/kind/kind-config.yaml b/contribute/kind/kind-config.yaml new file mode 100644 index 0000000..62fff9a --- /dev/null +++ b/contribute/kind/kind-config.yaml @@ -0,0 +1,21 @@ +--- +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 8080 + protocol: TCP + - containerPort: 443 + hostPort: 8443 + protocol: TCP + extraMounts: + - hostPath: ./ + containerPath: /app diff --git a/contribute/kind/traefik-values.yaml b/contribute/kind/traefik-values.yaml new file mode 100644 index 0000000..6fae1b7 --- /dev/null +++ b/contribute/kind/traefik-values.yaml @@ -0,0 +1,17 @@ +--- +providers: + kubernetesCRD: + enabled: false + kubernetesIngress: + namespaces: + - default + - ingress-traefik + +ports: + web: + hostPort: 80 + websecure: + hostPort: 443 + +service: + type: ClusterIP \ No newline at end of file diff --git a/contribute/midaas-ws/Dockerfile b/contribute/midaas-ws/Dockerfile new file mode 100644 index 0000000..a55e887 --- /dev/null +++ b/contribute/midaas-ws/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.9 +WORKDIR /app +COPY ./requirements.txt . +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt +COPY main.py . +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--reload"] + diff --git a/contribute/midaas-ws/main.py b/contribute/midaas-ws/main.py new file mode 100644 index 0000000..3e3734a --- /dev/null +++ b/contribute/midaas-ws/main.py @@ -0,0 +1,125 @@ +import os +import pathlib +import json +from fastapi import FastAPI, Response, Request, Body +from pydantic import BaseModel +import logging + +logger = logging.getLogger('uvicorn.error') + +KEYNAME = "ddns-key." + os.environ.get("MIDAAS_KEYNAME", "d1") +KEYVALUE = os.environ.get("MIDAAS_KEYVALUE", "test") +ZONES = os.environ.get("MIDAAS_ZONES", "d1.dev.local") +ALL_ZONES = ZONES.split(",") + +app = FastAPI() + + +class TTLDelete(BaseModel): + keyname: str + keyvalue: str + + +class TTLCreate(BaseModel): + ttl: int + keyname: str + keyvalue: str + + +def check_TSIG(keyname, keyvalue): + if not keyname == KEYNAME and KEYVALUE == keyvalue: + logger.info(f"Keyname or Keyvalue not match") + logger.info(f"Keyname: {keyname} with {KEYNAME}") + logger.info(f"Keyvalue: {keyname} with {KEYNAME}") + return False + return True + + +def create_zone(file): + logger.info(f"Creating zone on {file}") + with open(file, "w+") as f: + json.dump({}, f) + + +def create_zone_if_not_exist(file): + p = pathlib.Path(file) + if not p.exists(): + create_zone(file) + else: + # check if zone is not empty + with open(file, "r") as f: + content = f.read() + if not content: + create_zone(file) + + +def read_zone(file): + with open(file, "r") as f: + data = json.loads(f.read()) + return data + + +@app.get("/ws/{domaine}") +async def list_domain(request: Request, domaine): + logger.info(f"GET on url: {request.url}") + records = {} + for zone in ALL_ZONES: + if zone in domaine.strip().lower(): + file = f"/tmp/{zone}" + create_zone_if_not_exist(file) + records = read_zone(file) + logger.info(f"Zone content : {records}") + return records + return records + + +@app.get("/healthz") +async def health(): + return {"status": "OK"} + + +@app.put("/ws/{domaine}/{type}/{valeur}") +def create(response: Response, request: Request, domaine: str, type: str, valeur: str, TTL: TTLCreate) -> dict: + logger.info(f"PUT on url: {request.url}") + if not check_TSIG(keyname=TTL.keyname, keyvalue=TTL.keyvalue): + return {"status": "ERROR", "message": "wrong credentials"} + + for zone in ALL_ZONES: + if zone in domaine: + file = f"/tmp/{zone}" + create_zone_if_not_exist(file=file) + data = read_zone(file=file) + with open(file, "w+") as f: + if type == "CNAME": + valeur += "." + key = f"{domaine}./{type}/{valeur}" + updated_data = data | {key: {"type": type, + "valeur": valeur, + "ttl": TTL.ttl}} + json.dump(updated_data, f) + logger.info(f"Zone content : {updated_data}") + return {"status": "OK"} + return {"status": "ERROR", "message": "zone not available"} + + +@app.delete("/ws/{domaine}/{type}/{valeur}") +def delete(response: Response, request: Request, domaine: str, type: str, valeur: str, TTL: TTLDelete) -> dict: + logger.info(f"DELETE on url: {request.url}") + if not check_TSIG(keyname=TTL.keyname, keyvalue=TTL.keyvalue): + return {"status": "ERROR", "message": "wrong credentials"} + + for zone in ALL_ZONES: + if zone in domaine: + file = f"/tmp/{zone}" + create_zone_if_not_exist(file=file) + data = read_zone(file=file) + with open(file, "w") as f: + if type == "CNAME": + valeur += "." + key = f"{domaine}./{type}/{valeur}" + if key in data: + data.pop(key) + json.dump(data, f) + logger.info(f"Zone content : {data}") + return {"status": "OK"} + return {"status": "ERROR", "message": "no domain"} diff --git a/contribute/midaas-ws/requirements.txt b/contribute/midaas-ws/requirements.txt new file mode 100644 index 0000000..42612ea --- /dev/null +++ b/contribute/midaas-ws/requirements.txt @@ -0,0 +1,3 @@ +fastapi +requests +uvicorn diff --git a/contribute/ressources/external-dns-values.yaml b/contribute/ressources/external-dns-values.yaml new file mode 100644 index 0000000..287bd60 --- /dev/null +++ b/contribute/ressources/external-dns-values.yaml @@ -0,0 +1,42 @@ +# -- How DNS records are synchronized between sources and providers; available values are `sync` & `upsert-only`. +policy: sync +# -- Specify the registry for storing ownership and labels. +# Valid values are `txt`, `aws-sd`, `dynamodb` & `noop`. +# If `noop` midaas manage all records on zone +registry: noop +# can restrict zone +provider: + name: webhook + webhook: + image: + repository: external-dns-midaas-webhook + tag: dev + pullPolicy: Always + env: + - name: GOFLAGS + value: -buildvcs=false + - name: TSIG_ZONE_${MIDAAS_ENV_KEYNAME} + value: ${MIDAAS_ENV_KEYVALUE} + - name: API_LOG_LEVEL + value: DEBUG + - name: PROVIDER_DNS_ZONE_SUFFIX + value: ${MIDAAS_DEV_SUFFIX} + - name: PROVIDER_WS_URL + value: ${MIDAAS_WS_URL} + - name: https_proxy + value: ${https_proxy} + - name: no_proxy + value: .default,{no_proxy} + extraVolumeMounts: + - name: dev-workspace + mountPath: /go/src/app +podSecurityContext: + runAsNonRoot: false +extraVolumes: +- name: dev-workspace + hostPath: + path: /app + +sources: + - ingress +logLevel: debug \ No newline at end of file diff --git a/contribute/ressources/ingress.yaml b/contribute/ressources/ingress.yaml new file mode 100644 index 0000000..5637007 --- /dev/null +++ b/contribute/ressources/ingress.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test +spec: + rules: + - host: test.d1.dev.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: midaas + port: + number: 8080 \ No newline at end of file diff --git a/midaas/midaas.go b/midaas/midaas.go index 523ce1c..479df2a 100644 --- a/midaas/midaas.go +++ b/midaas/midaas.go @@ -229,9 +229,9 @@ func RequestUrl(method string, url string, body Body) error { bodyResponse, err := io.ReadAll(response.Body) defer response.Body.Close() - if err != nil { - return err - } + if err != nil { + return err + } if response.StatusCode > 200 { return fmt.Errorf(string(bodyResponse)) }