From 9f1e129cd45094efa3ad1b56d8e7698d0e9da384 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 8 Feb 2024 09:48:07 +0100 Subject: [PATCH 1/7] fix: change k8s port to 5800 --- docs/source/deployment/index.rst | 2 +- .../api/0.8/11-container-images-k8s.ipynb | 2 +- .../helm/syft/templates/registry-service.yaml | 2 +- .../syft/templates/registry-statefulset.yaml | 2 +- scripts/build_images.sh | 2 +- tox.ini | 24 +++++++++---------- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/source/deployment/index.rst b/docs/source/deployment/index.rst index 461ecb3734e..866393b24f5 100644 --- a/docs/source/deployment/index.rst +++ b/docs/source/deployment/index.rst @@ -380,7 +380,7 @@ Ingress is not working on Mac and Docker and the issue is `being tracked here 22/tcp, 127.0.0.1:57955->2376/tcp, 127.0.0.1:57957->5000/tcp, 127.0.0.1:57958->8443/tcp, 127.0.0.1:57956->32443/tcp minikube + 57f73851bf08 gcr.io/k8s-minikube/kicbase:v0.0.25 "/usr/local/bin/entr…" 46 hours ago Up About a minute 127.0.0.1:57954->22/tcp, 127.0.0.1:57955->2376/tcp, 127.0.0.1:57957->5800/tcp, 127.0.0.1:57958->8443/tcp, 127.0.0.1:57956->32443/tcp minikube diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index c28b0187567..6c57608e198 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -210,7 +210,7 @@ "metadata": {}, "outputs": [], "source": [ - "registry = \"k3d-registry.localhost:5000\"\n", + "registry = \"k3d-registry.localhost:5800\"\n", "repo = \"openmined/grid-backend\"\n", "\n", "res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n", diff --git a/packages/grid/helm/syft/templates/registry-service.yaml b/packages/grid/helm/syft/templates/registry-service.yaml index f96060e3a4d..284ef1f7292 100644 --- a/packages/grid/helm/syft/templates/registry-service.yaml +++ b/packages/grid/helm/syft/templates/registry-service.yaml @@ -11,7 +11,7 @@ spec: ports: - protocol: TCP port: 80 - targetPort: 5000 + targetPort: 5800 selector: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/component: registry diff --git a/packages/grid/helm/syft/templates/registry-statefulset.yaml b/packages/grid/helm/syft/templates/registry-statefulset.yaml index c4fb60d474d..e5d6171b438 100644 --- a/packages/grid/helm/syft/templates/registry-statefulset.yaml +++ b/packages/grid/helm/syft/templates/registry-statefulset.yaml @@ -28,7 +28,7 @@ spec: - name: REGISTRY_STORAGE_DELETE_ENABLED value: "true" ports: - - containerPort: 5000 + - containerPort: 5800 volumeMounts: - mountPath: /var/lib/registry name: registry-data diff --git a/scripts/build_images.sh b/scripts/build_images.sh index 280ca544620..da58203e08b 100644 --- a/scripts/build_images.sh +++ b/scripts/build_images.sh @@ -1,6 +1,6 @@ #!/bin/bash -REGISTRY=${1:-"k3d-registry.localhost:5000"} +REGISTRY=${1:-"k3d-registry.localhost:5800"} TAG=${2:-"latest"} docker image build -f ./packages/grid/backend/backend.dockerfile --target backend -t $REGISTRY/openmined/grid-backend:$TAG ./packages diff --git a/tox.ini b/tox.ini index 0362e12bfcb..aa9f82abad7 100644 --- a/tox.ini +++ b/tox.ini @@ -693,7 +693,7 @@ commands = # Creating registory - bash -c 'k3d registry create registry.localhost --port 5000 -v `pwd`/k3d-registry:/var/lib/registry || true' + bash -c 'k3d registry create registry.localhost --port 5800 -v `pwd`/k3d-registry:/var/lib/registry || true' # Creating testgateway1 cluster on port 9081 bash -c 'NODE_NAME=testgateway1 NODE_PORT=9081 && \ @@ -707,7 +707,7 @@ commands = -p gateway \ --var NODE_NAME=$NODE_NAME \ --var TEST_MODE=1 \ - --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 \ + --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 \ --var NODE_TYPE=gateway \ deploy -b; \ do ((--r))||exit;echo "retrying" && sleep 20;done)' @@ -723,7 +723,7 @@ commands = devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \ --var NODE_NAME=$NODE_NAME \ --var TEST_MODE=1 \ - --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 \ + --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 \ deploy -b; \ do ((--r))||exit;echo "retrying" && sleep 20;done)' @@ -869,14 +869,14 @@ commands = bash -c "k3d registry delete k3d-registry.localhost || true" - bash -c 'k3d registry create registry.localhost --port 5000 -v `pwd`/k3d-registry:/var/lib/registry || true' + bash -c 'k3d registry create registry.localhost --port 5800 -v `pwd`/k3d-registry:/var/lib/registry || true' bash -c 'NODE_NAME=syft NODE_PORT=8080 && \ k3d cluster create syft -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \ k3d cluster start syft' - ; skopeo list-tags --tls-verify=false docker://k3d-registry.localhost:5000/openmined/grid-backend - ; skopeo inspect --tls-verify=false docker://k3d-registry.localhost:5000/openmined/grid-backend:f1725f + ; skopeo list-tags --tls-verify=false docker://k3d-registry.localhost:5800/openmined/grid-backend + ; skopeo inspect --tls-verify=false docker://k3d-registry.localhost:5800/openmined/grid-backend:f1725f ; helm uninstall --kube-context k3d-syft --namespace syft syft ; helm install --kube-context k3d-syft --namespace syft syft ./syft ; k3d cluster create syft -p "8080:80@loadbalancer" && k3d cluster start syft @@ -932,13 +932,13 @@ allowlist_externals = sudo commands = ; create registry - bash -c 'k3d registry create registry.localhost --port 5000 -v $HOME/.k3d-registry:/var/lib/registry || true' + bash -c 'k3d registry create registry.localhost --port 5800 -v $HOME/.k3d-registry:/var/lib/registry || true' ; add patches to host bash -c 'if ! grep -q k3d-registry.localhost /etc/hosts; then sudo {envpython} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; fi' ; Fail this command if registry is not working - bash -c 'URL=http://k3d-registry.localhost:5000/v2/_catalog; curl -X GET $URL' + bash -c 'URL=http://k3d-registry.localhost:5800/v2/_catalog; curl -X GET $URL' [testenv:dev.k8s.start] description = Start local Kubernetes registry & cluster with k3d @@ -953,7 +953,7 @@ commands = tox -e dev.k8s.registry ; for NodePort to work add the following --> -p "NodePort:NodePort@loadbalancer" - bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5000; \ + bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5800; \ kubectl create namespace syft || true' ; dump cluster info @@ -968,7 +968,7 @@ allowlist_externals = bash commands = ; deploy syft helm charts - bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000' + bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800' [testenv:dev.k8s.hotreload] description = Start development with hot-reload in Kubernetes @@ -979,7 +979,7 @@ allowlist_externals = tox commands = ; deploy syft helm charts with hot-reload - bash -c 'devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000' + bash -c 'devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800' [testenv:dev.k8s.info] description = Gather info about the localKubernetes cluster @@ -1003,7 +1003,7 @@ allowlist_externals = bash commands = bash -c 'devspace purge --force-purge --kube-context k3d-syft-dev --namespace syft; sleep 3' - bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 || true' + bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 || true' bash -c 'kubectl config use-context k3d-syft-dev' bash -c 'kubectl delete all --all --namespace syft || true' bash -c 'kubectl delete pvc --all --namespace syft || true' From 69ae2740bcfc98c2a65bc7103ddee3331c7bdf28 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 8 Feb 2024 17:46:09 +0530 Subject: [PATCH 2/7] [helm] fix container port --- packages/grid/helm/syft/templates/registry-service.yaml | 2 +- packages/grid/helm/syft/templates/registry-statefulset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/grid/helm/syft/templates/registry-service.yaml b/packages/grid/helm/syft/templates/registry-service.yaml index 284ef1f7292..f96060e3a4d 100644 --- a/packages/grid/helm/syft/templates/registry-service.yaml +++ b/packages/grid/helm/syft/templates/registry-service.yaml @@ -11,7 +11,7 @@ spec: ports: - protocol: TCP port: 80 - targetPort: 5800 + targetPort: 5000 selector: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/component: registry diff --git a/packages/grid/helm/syft/templates/registry-statefulset.yaml b/packages/grid/helm/syft/templates/registry-statefulset.yaml index e5d6171b438..c4fb60d474d 100644 --- a/packages/grid/helm/syft/templates/registry-statefulset.yaml +++ b/packages/grid/helm/syft/templates/registry-statefulset.yaml @@ -28,7 +28,7 @@ spec: - name: REGISTRY_STORAGE_DELETE_ENABLED value: "true" ports: - - containerPort: 5800 + - containerPort: 5000 volumeMounts: - mountPath: /var/lib/registry name: registry-data From 0243c232a84770dbccd903d940d8d88e9126223b Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Tue, 13 Feb 2024 12:09:43 +0100 Subject: [PATCH 3/7] fix: patch coredns so k3d registry with custom port is accessible inside cluster --- scripts/k8s-coredns-custom.yml | 12 ++++++++++++ tox.ini | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 scripts/k8s-coredns-custom.yml diff --git a/scripts/k8s-coredns-custom.yml b/scripts/k8s-coredns-custom.yml new file mode 100644 index 00000000000..0c1085ae2ee --- /dev/null +++ b/scripts/k8s-coredns-custom.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom + namespace: kube-system +data: + k3d-registry.localhost.server: | + k3d-registry.localhost { + log + rewrite name k3d-registry.localhost host.k3d.internal + forward . 127.0.0.1 + } diff --git a/tox.ini b/tox.ini index aa9f82abad7..d6d8ac01ca8 100644 --- a/tox.ini +++ b/tox.ini @@ -940,6 +940,19 @@ commands = ; Fail this command if registry is not working bash -c 'URL=http://k3d-registry.localhost:5800/v2/_catalog; curl -X GET $URL' +[testenv:dev.k8s.patch.coredns] +description = Patch CoreDNS to resolve k3d-registry.localhost +changedir = {toxinidir} +passenv=HOME,USER +allowlist_externals = + bash +commands = + ; patch coredns so k3d-registry.localhost works in k3d + bash -c 'kubectl apply -f ./scripts/k8s-coredns-custom.yml' + + ; restarts coredns + bash -c 'kubectl delete pod -n kube-system -l k8s-app=kube-dns' + [testenv:dev.k8s.start] description = Start local Kubernetes registry & cluster with k3d changedir = {toxinidir} @@ -956,6 +969,9 @@ commands = bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5800; \ kubectl create namespace syft || true' + ; patch coredns + tox -e dev.k8s.patch.coredns + ; dump cluster info tox -e dev.k8s.info From 2e50b74ac0298d1fbe5de6b7aa891578050937a3 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Tue, 13 Feb 2024 13:30:40 +0100 Subject: [PATCH 4/7] test: patch ci --- .github/workflows/pr-tests-stack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml index 67f0d4854b1..aa399de0019 100644 --- a/.github/workflows/pr-tests-stack.yml +++ b/.github/workflows/pr-tests-stack.yml @@ -629,6 +629,7 @@ jobs: curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace chmod +x devspace devspace version + tox -e dev.k8s.patch.coredns tox -e stack.test.integration.k8s tox -e syft.build.helm tox -e syft.package.helm From 69b9bf3da493509e9e6b7fdf30ece4cf0e7a5536 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Tue, 13 Feb 2024 14:14:16 +0100 Subject: [PATCH 5/7] test: patch integration tests for k8s --- .github/workflows/pr-tests-stack.yml | 1 - tox.ini | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml index aa399de0019..67f0d4854b1 100644 --- a/.github/workflows/pr-tests-stack.yml +++ b/.github/workflows/pr-tests-stack.yml @@ -629,7 +629,6 @@ jobs: curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace chmod +x devspace devspace version - tox -e dev.k8s.patch.coredns tox -e stack.test.integration.k8s tox -e syft.build.helm tox -e syft.package.helm diff --git a/tox.ini b/tox.ini index c1146456d76..20417f10c24 100644 --- a/tox.ini +++ b/tox.ini @@ -669,6 +669,7 @@ allowlist_externals = kubectx k3d echo + tox setenv = ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s} NODE_PORT = {env:NODE_PORT:9082} @@ -692,8 +693,8 @@ commands = bash -c "docker volume rm k3d-testdomain1-images --force || true" - # Creating registory - bash -c 'k3d registry create registry.localhost --port 5800 -v `pwd`/k3d-registry:/var/lib/registry || true' + # Creating registry + bash -c 'k3d registry create registry.localhost --port 5800 -v `pwd`/k3d-registry:/var/lib/registry || true' # Creating testgateway1 cluster on port 9081 bash -c 'NODE_NAME=testgateway1 NODE_PORT=9081 && \ @@ -735,6 +736,9 @@ commands = sleep 30 + # Patches CoreDNS + tox -e dev.k8s.patch.coredns + # wait for front end bash packages/grid/scripts/wait_for.sh service frontend --context k3d-testdomain1 --namespace testdomain1 bash -c '(kubectl logs service/frontend --context k3d-testdomain1 --namespace testdomain1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true' From cf3ec3ea82fcc1239497131441fca52c9e6894e8 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Tue, 13 Feb 2024 16:52:35 +0100 Subject: [PATCH 6/7] fix: doc sed --- docs/source/deployment/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deployment/index.rst b/docs/source/deployment/index.rst index 866393b24f5..461ecb3734e 100644 --- a/docs/source/deployment/index.rst +++ b/docs/source/deployment/index.rst @@ -380,7 +380,7 @@ Ingress is not working on Mac and Docker and the issue is `being tracked here 22/tcp, 127.0.0.1:57955->2376/tcp, 127.0.0.1:57957->5800/tcp, 127.0.0.1:57958->8443/tcp, 127.0.0.1:57956->32443/tcp minikube + 57f73851bf08 gcr.io/k8s-minikube/kicbase:v0.0.25 "/usr/local/bin/entr…" 46 hours ago Up About a minute 127.0.0.1:57954->22/tcp, 127.0.0.1:57955->2376/tcp, 127.0.0.1:57957->5000/tcp, 127.0.0.1:57958->8443/tcp, 127.0.0.1:57956->32443/tcp minikube From 51633f354e56ef04a40351c8ec941ffaeab3a27e Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Tue, 13 Feb 2024 17:00:57 +0100 Subject: [PATCH 7/7] test: k8s patch coredns after test cluster creation --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 20417f10c24..60e96d12b0c 100644 --- a/tox.ini +++ b/tox.ini @@ -718,6 +718,9 @@ commands = k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \ k3d cluster start $NODE_NAME' + # Patches CoreDNS + tox -e dev.k8s.patch.coredns + bash -c 'NODE_NAME=testdomain1 NODE_PORT=9082 && \ cd packages/grid && \ (r=5;while ! \ @@ -736,9 +739,6 @@ commands = sleep 30 - # Patches CoreDNS - tox -e dev.k8s.patch.coredns - # wait for front end bash packages/grid/scripts/wait_for.sh service frontend --context k3d-testdomain1 --namespace testdomain1 bash -c '(kubectl logs service/frontend --context k3d-testdomain1 --namespace testdomain1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'