Merge branch 'dev' into fix/dataset_assets_contributor_uploader

.github/workflows/cd-syft-dev.yml

@@ -1,32 +1,49 @@
-name: CD - Syft - Deploy `dev` to K8s
+name: CD - Kubernetes - Dev
 
 on:
   schedule:
     - cron: "0 */3 * * *"
 
   workflow_dispatch:
     inputs:
-      none:
-        description: "Run Tests Manually"
-        required: false
+      check-cache:
+        type: boolean
+        description: "Check workflow cache"
+        default: false
+      deploy-helm:
+        type: boolean
+        description: "Deploy Helm Charts"
+        default: false
 
 jobs:
-  deploy-syft-dev:
+  check-last-run:
     runs-on: om-ci-16vcpu-ubuntu2204
+
+    outputs:
+      same-commit: ${{ steps.cache.outputs.cache-hit }}
+
     steps:
       - name: Check for new changes
+        if: github.event_name == 'schedule' || github.event.inputs.check-cache == 'true'
         id: cache
-        if: github.event_name == 'schedule'
         uses: actions/cache@v3
         with:
-          path: scripts/commit_hash # we don't care about the file, just the key
-          key: dev-commit-${{ github.sha }}
+          path: scripts/k8s_dev_hash # we don't care about the file, just the key
+          key: k8s-dev-${{ github.sha }}
           lookup-only: true
 
-      - name: Quit if no new changes
-        if: github.event_name == 'schedule' && steps.cache.outputs.cache-hit == 'true'
-        run: exit 0
+      - name: Save Commit SHA
+        if: github.event_name == 'schedule' || github.event.inputs.check-cache == 'true'
+        # only for making the cache action happy
+        run: echo "${{ github.sha }}" > scripts/k8s_dev_hash
+
+  deploy-syft-dev:
+    needs: check-last-run
+    if: needs.check-last-run.outputs.same-commit != 'true'
+
+    runs-on: om-ci-16vcpu-ubuntu2204
 
+    steps:
       - name: Permission to home directory
         run: |
           sudo chown -R $USER:$USER $HOME
@@ -51,10 +68,15 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Install Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az version
+
       - name: Login to Azure CLI
         uses: azure/login@v1
         with:
-          creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }}}
+          creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }}
 
       - name: Login to Azure Container Registry
         uses: azure/docker-login@v1
@@ -69,7 +91,7 @@ jobs:
         run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
 
       - name: Build and push `grid-backend` image to registry
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: ./packages
           file: ./packages/grid/backend/backend.dockerfile
@@ -78,21 +100,23 @@ jobs:
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build and push `grid-frontend` image to registry
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/frontend
           file: ./packages/grid/frontend/frontend.dockerfile
           push: true
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }}
           target: grid-ui-development
 
       # TODO: Re-enable once we have Enclave up and running
       # - name: Build and push `grid-enclave` image to registry
-      #   uses: docker/build-push-action@v4
+      #   uses: docker/build-push-action@v5
       #   with:
       #     context: ./packages
       #     file: ./packages/grid/worker/worker.dockerfile
@@ -101,38 +125,43 @@ jobs:
       #     tags: |
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }}
+      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build and push `grid-headscale` image to registry
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/vpn
           file: ./packages/grid/vpn/headscale.dockerfile
           push: true
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build and push `grid-tailscale` image to registry
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/vpn
           file: ./packages/grid/vpn/tailscale.dockerfile
           push: true
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build and push `grid-vpn-iptables` image to registry
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/vpn
           file: ./packages/grid/vpn/iptables.dockerfile
           push: true
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build Helm Chart & Copy to infra
+        if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
         shell: bash
         run: |
           K3D_VERSION=v5.6.0
@@ -159,6 +188,7 @@ jobs:
           cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/dev/
 
       - name: Commit & push changes to Infra Repo
+        if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
         uses: EndBug/add-and-commit@v9
         with:
           author_name: ${{ secrets.OM_BOT_NAME }}
@@ -169,22 +199,27 @@ jobs:
           cwd: "./infrastructure/"
 
       - name: Cleanup Azure Container Registry
-        uses: azure/CLI@v1
-        with:
-          # SKIP_LINES = latest version dev & dev-<sha> (2 lines) + keep "n" previous version (n lines) + 1
-          inlineScript: |
-            ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }}
-            REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv)
+        run: |
+          ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }}
 
-            KEEP_PREV_VERSIONS=1
-            TAIL_FROM_LINE=$((2 + $KEEP_PREV_VERSIONS + 1))
+          echo ">> Fetching repo list.."
+          REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv)
 
-            for repo in $REPO_LIST
-            do
-                echo "Cleaning up '$repo'"
-                az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv | tail -n +$TAIL_FROM_LINE | xargs -r -I% az acr repository delete --name $ACR_REGISTRY_NAME --image $repo:% --yes
-            done
+          KEEP_PREV_VERSIONS=5
+          TAIL_FROM_LINE=$(($KEEP_PREV_VERSIONS + 1))
 
-      - name: Save Commit SHA
-        # only for cache to hit
-        run: echo "${{ github.sha }}" > scripts/commit_hash
+          for repo in $REPO_LIST
+          do
+              echo "Cleaning up '$repo'"
+              az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv \
+                | grep dev- \
+                | tail -n +$TAIL_FROM_LINE \
+                | xargs -r -I% az acr repository untag --name $ACR_REGISTRY_NAME --image $repo:%
+          done
+
+      - name: Logout and cleanup Azure account
+        if: always()
+        run: |
+          az logout
+          az cache purge
+          az account clear

packages/grid/helm/helm.py

@@ -1,6 +1,7 @@
 # stdlib
 import argparse
 import os
+import shutil
 import sys
 from typing import Any
 
@@ -83,9 +84,9 @@ def fix_devspace_yaml(d: Any) -> None:
             fix_devspace_yaml(item)
 
 
-def get_yaml_name(doc: Any) -> Any:
+def get_yaml_name(doc: dict) -> Any:
     try:
-        return yaml.safe_load(doc).get("metadata", {}).get("name", "")
+        return doc.get("metadata", {}).get("name", "")
     except Exception:  # nosec
         return ""
 
@@ -97,13 +98,12 @@ def main() -> None:
         "file", nargs="?", type=argparse.FileType("r"), default=sys.stdin
     )
     args = parser.parse_args()
-    helm_dir = "helm"
     text = args.file.read()
-    file_count = 0
 
-    # input_file = f"{helm_dir}/raw_manifests.yaml"
-    # with open(input_file, "w") as f:
-    #     f.write(text)
+    file_count = 0
+    helm_dir = "helm"
+    manifest_file = f"{helm_dir}/manifests.yaml"
+    helm_chart_template_dir = f"{helm_dir}/syft/templates"
 
     # Read input from file or stdin
     lines = text.splitlines()
@@ -113,50 +113,42 @@ def main() -> None:
         first_index = next(
             i for i, line in enumerate(lines) if line.strip().startswith("apiVersion")
         )
-        input_data = "---\n" + "\n".join(lines[first_index - 1 :])
+        input_data = "\n".join(lines[first_index:])
     except StopIteration:
         print("❌ Error: No line starting with 'apiVersion' found in the input.")
-        print("------------------------------")
-        print("Got input text:")
-        print(text)
-        print("------------------------------")
         exit(1)
 
-    helm_chart_template_dir = f"{helm_dir}/syft/templates"
+    # Load the multi-doc yaml file
+    try:
+        yaml_docs = list(yaml.safe_load_all(input_data))
+    except Exception as e:
+        print(f"❌ Error while parsing yaml file: {e}")
+        exit(1)
+
+    # clear templates dir
+    shutil.rmtree(helm_chart_template_dir, ignore_errors=True)
 
-    # Split input_data into separate documents
-    yaml_docs = input_data.split("---")
+    # Create directories if they don't exist
+    os.makedirs(helm_chart_template_dir, exist_ok=True)
+
+    # Cleanup YAML docs
+    yaml_docs = [doc for doc in yaml_docs if doc]
 
     # Sort YAML docs based on metadata name
     yaml_docs.sort(key=get_yaml_name)
 
-    # Join sorted YAML docs
-    sorted_input_data = "---".join(yaml_docs)
-
     # Save sorted YAML docs to file
-    input_file = f"{helm_dir}/manifests.yaml"
-    with open(input_file, "w") as f:
-        f.write(sorted_input_data)
+    with open(manifest_file, "w") as f:
+        yaml.dump_all(yaml_docs, f)
 
     for doc in yaml_docs:
-        lines = doc.strip().split("\n")
-        if len(lines) <= 2:
-            continue  # skip empty sections
-
-        output_dir = os.path.join(helm_chart_template_dir)
-
-        # Create directories if they don't exist
-        os.makedirs(output_dir, exist_ok=True)
-
-        # Parse yaml to find metadata.name
-        yaml_content = yaml.safe_load("\n".join(lines))  # exclude source_line
-        fix_devspace_yaml(yaml_content)
-        name = yaml_content.get("metadata", {}).get("name")
-        kind = yaml_content.get("kind", "").lower()
+        fix_devspace_yaml(doc)
+        name = doc.get("metadata", {}).get("name")
+        kind = doc.get("kind", "").lower()
         if name:
             # Create new file with name or append if it already exists
-            new_file = os.path.join(output_dir, f"{name}-{kind}.yaml")
-            yaml_dump = yaml.dump(yaml_content)
+            new_file = os.path.join(helm_chart_template_dir, f"{name}-{kind}.yaml")
+            yaml_dump = yaml.dump(doc)
             yaml_dump = (
                 yaml_dump.replace("'{{", "{{")
                 .replace("}}'", "}}")
@@ -171,7 +163,7 @@ def main() -> None:
     if file_count > 0:
         print(f"✅ Done: Generated {file_count} template files")
     else:
-        print("❌ Failed: Generated zero files. Check input file for errors.")
+        print("❌ Failed: No files were generated. Check input for errors.")
         exit(1)
 
 

tox.ini

@@ -755,23 +755,25 @@ allowlist_externals =
     helm
 commands =
     k3d version
+    devspace version
 
-    ; bash -c "docker rm $(docker ps -aq) --force || true"
-    bash -c "k3d cluster delete build || true && \
+    bash -c "k3d cluster delete build ; \
         docker volume rm k3d-build-images --force || true"
 
-    bash -c 'k3d cluster create build || true && \
-        k3d cluster start build'
-
-    bash -c 'rm -rf packages/grid/helm/syft/templates/ && mkdir -p packages/grid/helm/syft/templates/'
+    bash -c 'k3d cluster create build && \
+        k3d cluster start build && \
+        echo "Waiting for cluster to be ready..." && \
+        sleep 20'
 
     bash -c 'cd packages/grid && \
         [[ -n "$CONTAINER_REGISTRY" ]] && REGISTRY_FLAG="--var CONTAINER_REGISTRY=$CONTAINER_REGISTRY" || REGISTRY_FLAG="" && \
         [[ -n "$VERSION" ]] && VERSION_FLAG="--var VERSION=$VERSION" || VERSION_FLAG="" && \
-        devspace deploy --render --skip-build --no-warn --silent ${REGISTRY_FLAG} ${VERSION_FLAG} --kube-context "k3d-build" > out.txt'
+        devspace deploy --render --skip-build ${REGISTRY_FLAG} ${VERSION_FLAG} --kube-context "k3d-build" --no-warn --no-colors > out.txt ; \
+        EXITCODE=$?; OUTPUT=$(cat out.txt); printf "Devspace exit code: $EXITCODE\nDevspace output:\n$OUTPUT\n"; exit $EXITCODE'
 
     bash -c 'cd packages/grid && \
-        python3 helm/helm.py out.txt'
+        python3 helm/helm.py out.txt && \
+        rm out.txt'
 
     bash -c 'cd packages/grid/helm && \
         helm lint syft'