diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index 68bc6982f8c..78f18284743 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -210,7 +210,7 @@ "metadata": {}, "outputs": [], "source": [ - "registry = \"k3d-registry.localhost:5000\"\n", + "registry = \"k3d-registry.localhost:5800\"\n", "repo = \"openmined/grid-backend\"\n", "\n", "res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n", diff --git a/scripts/build_images.sh b/scripts/build_images.sh index 280ca544620..da58203e08b 100644 --- a/scripts/build_images.sh +++ b/scripts/build_images.sh @@ -1,6 +1,6 @@ #!/bin/bash -REGISTRY=${1:-"k3d-registry.localhost:5000"} +REGISTRY=${1:-"k3d-registry.localhost:5800"} TAG=${2:-"latest"} docker image build -f ./packages/grid/backend/backend.dockerfile --target backend -t $REGISTRY/openmined/grid-backend:$TAG ./packages diff --git a/scripts/k8s-coredns-custom.yml b/scripts/k8s-coredns-custom.yml new file mode 100644 index 00000000000..0c1085ae2ee --- /dev/null +++ b/scripts/k8s-coredns-custom.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom + namespace: kube-system +data: + k3d-registry.localhost.server: | + k3d-registry.localhost { + log + rewrite name k3d-registry.localhost host.k3d.internal + forward . 127.0.0.1 + } diff --git a/tox.ini b/tox.ini index cb254f79ffc..60e96d12b0c 100644 --- a/tox.ini +++ b/tox.ini @@ -669,6 +669,7 @@ allowlist_externals = kubectx k3d echo + tox setenv = ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s} NODE_PORT = {env:NODE_PORT:9082} @@ -692,8 +693,8 @@ commands = bash -c "docker volume rm k3d-testdomain1-images --force || true" - # Creating registory - bash -c 'k3d registry create registry.localhost --port 5000 -v `pwd`/k3d-registry:/var/lib/registry || true' + # Creating registry + bash -c 'k3d registry create registry.localhost --port 5800 -v `pwd`/k3d-registry:/var/lib/registry || true' # Creating testgateway1 cluster on port 9081 bash -c 'NODE_NAME=testgateway1 NODE_PORT=9081 && \ @@ -707,7 +708,7 @@ commands = -p gateway \ --var NODE_NAME=$NODE_NAME \ --var TEST_MODE=1 \ - --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 \ + --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 \ --var NODE_TYPE=gateway \ deploy -b; \ do ((--r))||exit;echo "retrying" && sleep 20;done)' @@ -717,13 +718,16 @@ commands = k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \ k3d cluster start $NODE_NAME' + # Patches CoreDNS + tox -e dev.k8s.patch.coredns + bash -c 'NODE_NAME=testdomain1 NODE_PORT=9082 && \ cd packages/grid && \ (r=5;while ! \ devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \ --var NODE_NAME=$NODE_NAME \ --var TEST_MODE=1 \ - --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 \ + --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 \ deploy -b; \ do ((--r))||exit;echo "retrying" && sleep 20;done)' @@ -875,14 +879,14 @@ commands = bash -c "k3d registry delete k3d-registry.localhost || true" - bash -c 'k3d registry create registry.localhost --port 5000 -v `pwd`/k3d-registry:/var/lib/registry || true' + bash -c 'k3d registry create registry.localhost --port 5800 -v `pwd`/k3d-registry:/var/lib/registry || true' bash -c 'NODE_NAME=syft NODE_PORT=8080 && \ k3d cluster create syft -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \ k3d cluster start syft' - ; skopeo list-tags --tls-verify=false docker://k3d-registry.localhost:5000/openmined/grid-backend - ; skopeo inspect --tls-verify=false docker://k3d-registry.localhost:5000/openmined/grid-backend:f1725f + ; skopeo list-tags --tls-verify=false docker://k3d-registry.localhost:5800/openmined/grid-backend + ; skopeo inspect --tls-verify=false docker://k3d-registry.localhost:5800/openmined/grid-backend:f1725f ; helm uninstall --kube-context k3d-syft --namespace syft syft ; helm install --kube-context k3d-syft --namespace syft syft ./syft ; k3d cluster create syft -p "8080:80@loadbalancer" && k3d cluster start syft @@ -938,13 +942,26 @@ allowlist_externals = sudo commands = ; create registry - bash -c 'k3d registry create registry.localhost --port 5000 -v $HOME/.k3d-registry:/var/lib/registry || true' + bash -c 'k3d registry create registry.localhost --port 5800 -v $HOME/.k3d-registry:/var/lib/registry || true' ; add patches to host bash -c 'if ! grep -q k3d-registry.localhost /etc/hosts; then sudo {envpython} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; fi' ; Fail this command if registry is not working - bash -c 'URL=http://k3d-registry.localhost:5000/v2/_catalog; curl -X GET $URL' + bash -c 'URL=http://k3d-registry.localhost:5800/v2/_catalog; curl -X GET $URL' + +[testenv:dev.k8s.patch.coredns] +description = Patch CoreDNS to resolve k3d-registry.localhost +changedir = {toxinidir} +passenv=HOME,USER +allowlist_externals = + bash +commands = + ; patch coredns so k3d-registry.localhost works in k3d + bash -c 'kubectl apply -f ./scripts/k8s-coredns-custom.yml' + + ; restarts coredns + bash -c 'kubectl delete pod -n kube-system -l k8s-app=kube-dns' [testenv:dev.k8s.start] description = Start local Kubernetes registry & cluster with k3d @@ -959,9 +976,12 @@ commands = tox -e dev.k8s.registry ; for NodePort to work add the following --> -p "NodePort:NodePort@loadbalancer" - bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5000; \ + bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5800; \ kubectl create namespace syft || true' + ; patch coredns + tox -e dev.k8s.patch.coredns + ; dump cluster info tox -e dev.k8s.info @@ -974,7 +994,7 @@ allowlist_externals = bash commands = ; deploy syft helm charts - bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000' + bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800' [testenv:dev.k8s.hotreload] description = Start development with hot-reload in Kubernetes @@ -985,7 +1005,7 @@ allowlist_externals = tox commands = ; deploy syft helm charts with hot-reload - bash -c 'devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000' + bash -c 'devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800' [testenv:dev.k8s.info] description = Gather info about the localKubernetes cluster @@ -1009,7 +1029,7 @@ allowlist_externals = bash commands = bash -c 'devspace purge --force-purge --kube-context k3d-syft-dev --namespace syft; sleep 3' - bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 || true' + bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 || true' bash -c 'kubectl config use-context k3d-syft-dev' bash -c 'kubectl delete all --all --namespace syft || true' bash -c 'kubectl delete pvc --all --namespace syft || true'