Merge pull request #8469 from OpenMined/fix/k8s-port

fix: change k8s port to 5800

notebooks/api/0.8/11-container-images-k8s.ipynb

@@ -210,7 +210,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "registry = \"k3d-registry.localhost:5000\"\n",
+    "registry = \"k3d-registry.localhost:5800\"\n",
     "repo = \"openmined/grid-backend\"\n",
     "\n",
     "res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n",

scripts/build_images.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-REGISTRY=${1:-"k3d-registry.localhost:5000"}
+REGISTRY=${1:-"k3d-registry.localhost:5800"}
 TAG=${2:-"latest"}
 
 docker image build -f ./packages/grid/backend/backend.dockerfile --target backend -t $REGISTRY/openmined/grid-backend:$TAG ./packages

scripts/k8s-coredns-custom.yml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  k3d-registry.localhost.server: |
+    k3d-registry.localhost {
+        log
+        rewrite name k3d-registry.localhost host.k3d.internal
+        forward . 127.0.0.1
+    }

tox.ini

@@ -669,6 +669,7 @@ allowlist_externals =
     kubectx
     k3d
     echo
+    tox
 setenv =
     ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
     NODE_PORT = {env:NODE_PORT:9082}
@@ -692,8 +693,8 @@ commands =
     bash -c "docker volume rm k3d-testdomain1-images --force || true"
 
 
-    # Creating registory
-    bash -c 'k3d registry create registry.localhost --port 5000  -v `pwd`/k3d-registry:/var/lib/registry || true'
+    # Creating registry
+    bash -c 'k3d registry create registry.localhost --port 5800 -v `pwd`/k3d-registry:/var/lib/registry || true'
 
     # Creating testgateway1 cluster on port 9081
     bash -c 'NODE_NAME=testgateway1 NODE_PORT=9081 && \
@@ -707,7 +708,7 @@ commands =
         -p gateway \
         --var NODE_NAME=$NODE_NAME \
         --var TEST_MODE=1 \
-        --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 \
+        --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 \
         --var NODE_TYPE=gateway \
         deploy -b; \
         do ((--r))||exit;echo "retrying" && sleep 20;done)'
@@ -717,13 +718,16 @@ commands =
         k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
         k3d cluster start $NODE_NAME'
 
+    # Patches CoreDNS
+    tox -e dev.k8s.patch.coredns
+
     bash -c 'NODE_NAME=testdomain1 NODE_PORT=9082 && \
         cd packages/grid && \
         (r=5;while ! \
         devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
         --var NODE_NAME=$NODE_NAME \
         --var TEST_MODE=1 \
-        --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 \
+        --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 \
         deploy -b; \
         do ((--r))||exit;echo "retrying" && sleep 20;done)'
 
@@ -875,14 +879,14 @@ commands =
 
     bash -c "k3d registry delete k3d-registry.localhost || true"
 
-    bash -c 'k3d registry create registry.localhost --port 5000  -v `pwd`/k3d-registry:/var/lib/registry || true'
+    bash -c 'k3d registry create registry.localhost --port 5800  -v `pwd`/k3d-registry:/var/lib/registry || true'
 
     bash -c 'NODE_NAME=syft NODE_PORT=8080 && \
         k3d cluster create syft -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
         k3d cluster start syft'
 
-    ; skopeo list-tags --tls-verify=false docker://k3d-registry.localhost:5000/openmined/grid-backend
-    ; skopeo inspect --tls-verify=false docker://k3d-registry.localhost:5000/openmined/grid-backend:f1725f
+    ; skopeo list-tags --tls-verify=false docker://k3d-registry.localhost:5800/openmined/grid-backend
+    ; skopeo inspect --tls-verify=false docker://k3d-registry.localhost:5800/openmined/grid-backend:f1725f
     ; helm uninstall --kube-context k3d-syft --namespace syft syft
     ; helm install --kube-context k3d-syft --namespace syft syft ./syft
     ; k3d cluster create syft -p "8080:80@loadbalancer" && k3d cluster start syft
@@ -938,13 +942,26 @@ allowlist_externals =
     sudo
 commands =
     ; create registry
-    bash -c 'k3d registry create registry.localhost --port 5000 -v $HOME/.k3d-registry:/var/lib/registry || true'
+    bash -c 'k3d registry create registry.localhost --port 5800 -v $HOME/.k3d-registry:/var/lib/registry || true'
 
     ; add patches to host
     bash -c 'if ! grep -q k3d-registry.localhost /etc/hosts; then sudo {envpython} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; fi'
 
     ; Fail this command if registry is not working
-    bash -c 'URL=http://k3d-registry.localhost:5000/v2/_catalog; curl -X GET $URL'
+    bash -c 'URL=http://k3d-registry.localhost:5800/v2/_catalog; curl -X GET $URL'
+
+[testenv:dev.k8s.patch.coredns]
+description = Patch CoreDNS to resolve k3d-registry.localhost
+changedir = {toxinidir}
+passenv=HOME,USER
+allowlist_externals =
+    bash
+commands =
+    ; patch coredns so k3d-registry.localhost works in k3d
+    bash -c 'kubectl apply -f ./scripts/k8s-coredns-custom.yml'
+
+    ; restarts coredns
+    bash -c 'kubectl delete pod -n kube-system -l k8s-app=kube-dns'
 
 [testenv:dev.k8s.start]
 description = Start local Kubernetes registry & cluster with k3d
@@ -959,9 +976,12 @@ commands =
     tox -e dev.k8s.registry
 
     ; for NodePort to work add the following --> -p "NodePort:NodePort@loadbalancer"
-    bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5000; \
+    bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5800; \
         kubectl create namespace syft || true'
 
+    ; patch coredns
+    tox -e dev.k8s.patch.coredns
+
     ; dump cluster info
     tox -e dev.k8s.info
 
@@ -974,7 +994,7 @@ allowlist_externals =
     bash
 commands =
     ; deploy syft helm charts
-    bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000'
+    bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800'
 
 [testenv:dev.k8s.hotreload]
 description = Start development with hot-reload in Kubernetes
@@ -985,7 +1005,7 @@ allowlist_externals =
     tox
 commands =
     ; deploy syft helm charts with hot-reload
-    bash -c 'devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000'
+    bash -c 'devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800'
 
 [testenv:dev.k8s.info]
 description = Gather info about the localKubernetes cluster
@@ -1009,7 +1029,7 @@ allowlist_externals =
     bash
 commands =
     bash -c 'devspace purge --force-purge --kube-context k3d-syft-dev --namespace syft; sleep 3'
-    bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 || true'
+    bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 || true'
     bash -c 'kubectl config use-context k3d-syft-dev'
     bash -c 'kubectl delete all --all --namespace syft || true'
     bash -c 'kubectl delete pvc --all --namespace syft || true'