Merge pull request #8416 from OpenMined/yash/k8s-dev-fixes

Fix kubernetes dev workflows

packages/grid/devspace.yaml

@@ -6,9 +6,12 @@ pipelines:
   # This is the pipeline for the main command: `devspace dev` (or `devspace run-pipeline dev`)
   dev:
     run: |-
+      stop_dev --all
+      purge_deployments --all
       run_dependencies --all       # 1. Deploy any projects this project needs (see "dependencies")
       ensure_pull_secrets --all    # 2. Ensure pull secrets
       create_deployments --all     # 3. Deploy Helm charts and manifests specfied as "deployments"
+      build_images --all
       start_dev --all              # 4. Start dev mode "app" (see "dev" section)
   # You can run this pipeline via `devspace deploy` (or `devspace run-pipeline deploy`)
   deploy:
@@ -33,9 +36,7 @@ images:
     dockerfile: ./backend/backend.dockerfile
     context: ../
     tags:
-      - dev-latest
-      - "${VERSION}"
-      - "${devspace.git.commit}"
+      - dev-${DEVSPACE_TIMESTAMP}
   frontend:
     image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_FRONTEND}"
     buildKit:
@@ -44,9 +45,7 @@ images:
     target: "grid-ui-production"
     context: ./frontend
     tags:
-      - dev-latest
-      - "${VERSION}"
-      - "${devspace.git.commit}"
+      - dev-${DEVSPACE_TIMESTAMP}
   seaweedfs:
     image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_SEAWEEDFS}"
     buildKit: {}
@@ -55,9 +54,7 @@ images:
     dockerfile: ./seaweedfs/seaweedfs.dockerfile
     context: ./seaweedfs
     tags:
-      - dev-latest
-      - "${VERSION}"
-      - "${devspace.git.commit}"
+      - dev-${DEVSPACE_TIMESTAMP}
 
 # This is a list of `deployments` that DevSpace can create for this project
 deployments:
@@ -68,28 +65,34 @@ deployments:
       values:
         syft:
           registry: ${CONTAINER_REGISTRY}
-          version: "dev-latest"
+          version: dev-${DEVSPACE_TIMESTAMP}
         node:
           settings:
             nodeName: ${NODE_NAME}
             nodeType: "domain"
-        # configuration:
-        #   devmode: True
+        configuration:
+          devmode: True
 
 dev:
   mongo:
-    imageSelector: "${MONGO_IMAGE}:${MONGO_VERSION}"
+    labelSelector:
+      app.kubernetes.io/name: syft
+      app.kubernetes.io/component: mongo
     ports:
       - port: "27017"
   seaweedfs:
-    imageSelector: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_SEAWEEDFS}"
+    labelSelector:
+      app.kubernetes.io/name: syft
+      app.kubernetes.io/component: seaweedfs
     ports:
       - port: "9333" # admin
       - port: "8888" # filer
       - port: "8333" # S3
       - port: "4001" # mount azure
   backend:
-    imageSelector: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}"
+    labelSelector:
+      app.kubernetes.io/name: syft
+      app.kubernetes.io/component: backend
     env:
       - name: DEV_MODE
         value: "True"
@@ -104,17 +107,3 @@ profiles:
       - op: replace
         path: deployments.syft.helm.values.node.settings.nodeType
         value: "gateway"
-
-commands:
-  start:
-    command: |-
-      tox -e dev.k8s.start
-  deploy:
-    command: |-
-      devspace deploy --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:12345
-  dev:
-    command: |-
-      devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:12345
-  info:
-    command: |-
-      tox -e dev.k8s.info

tox.ini

@@ -3,7 +3,9 @@ envlist =
     dev.k8s.registry
     dev.k8s.start
     dev.k8s.deploy
+    dev.k8s.hotreload
     dev.k8s.info
+    dev.k8s.cleanup
     dev.k8s.destroy
     dev.k8s.destroyall
     hagrid.publish
@@ -919,8 +921,14 @@ allowlist_externals =
     bash
     sudo
 commands =
-    bash -c 'k3d registry create registry.localhost --port 12345 -v $HOME/.k3d-registry:/var/lib/registry || true'
-    sudo {envpython} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts
+    ; create registry
+    bash -c 'k3d registry create registry.localhost --port 5000 -v $HOME/.k3d-registry:/var/lib/registry || true'
+
+    ; add patches to host
+    bash -c 'if ! grep -q k3d-registry.localhost /etc/hosts; then sudo {envpython} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; fi'
+
+    ; Fail this command if registry is not working
+    bash -c 'URL=http://k3d-registry.localhost:5000/v2/_catalog; curl -X GET $URL'
 
 [testenv:dev.k8s.start]
 description = Start local Kubernetes registry & cluster with k3d
@@ -931,21 +939,37 @@ allowlist_externals =
     sleep
     tox
 commands =
+    ; start registry
     tox -e dev.k8s.registry
-    sleep 3
-    bash -c 'URL=http://k3d-registry.localhost:12345/v2/_catalog; curl -X GET $URL'
-    bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:12345; \
-        kubectl create namespace syft;'
+
+    ; for NodePort to work add the following --> -p "NodePort:NodePort@loadbalancer"
+    bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5000; \
+        kubectl create namespace syft || true'
+
+    ; dump cluster info
     tox -e dev.k8s.info
 
 [testenv:dev.k8s.deploy]
-description = Deploy to a local Kubernetes cluster with Devspace
+description = Deploy Syft to a local Kubernetes cluster with Devspace
+changedir = {toxinidir}/packages/grid
+passenv=HOME, USER
+allowlist_externals =
+    tox
+    bash
+commands =
+    ; deploy syft helm charts
+    bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000'
+
+[testenv:dev.k8s.hotreload]
+description = Start development with hot-reload in Kubernetes
 changedir = {toxinidir}/packages/grid
 passenv=HOME, USER
 allowlist_externals =
     bash
+    tox
 commands =
-    bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:12345'
+    ; deploy syft helm charts with hot-reload
+    bash -c 'devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000'
 
 [testenv:dev.k8s.info]
 description = Gather info about the localKubernetes cluster
@@ -961,15 +985,32 @@ commands =
     kubectl config current-context
     kubectl get namespaces
 
+[testenv:dev.k8s.cleanup]
+description = Cleanup Syft deployment and associated resources, but keep the cluster running
+changedir = {toxinidir}/packages/grid
+passenv=HOME, USER
+allowlist_externals =
+    bash
+commands =
+    bash -c 'devspace purge --force-purge --kube-context k3d-syft-dev --namespace syft'
+    bash -c 'kubectl delete job --all --namespace syft || true'
+    bash -c 'kubectl delete secret --all --namespace syft || true'
+    bash -c 'kubectl delete pvc --all --namespace syft || true'
+    bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 || true'
+
 [testenv:dev.k8s.destroy]
 description = Destroy local Kubernetes cluster
 changedir = {toxinidir}/packages/grid
 passenv=HOME, USER
 allowlist_externals =
+    tox
     bash
 commands =
+    ; purge deployment and dangling resources
+    tox -e dev.k8s.cleanup
+
+    ; destroy cluster
     bash -c '\
-        devspace purge --kube-context k3d-syft-dev --namespace syft; \
         rm -rf .devspace; echo ""; \
         k3d cluster delete syft-dev; echo ""; \
         kubectl config view'
@@ -982,12 +1023,14 @@ ignore_errors=True
 allowlist_externals =
     bash
     tox
-    sudo
     rm
 commands =
+    ; destroy cluster
     tox -e dev.k8s.destroy
+
+    ; destroy registry
     bash -c 'k3d registry delete registry.localhost || true'
-    sudo rm -rf ~/.k3d-registry
+    bash -c 'sudo rm -rf ~/.k3d-registry'
 
 [testenv:backend.test.basecpu]
 description = Base CPU Docker Image Test