Merge branch 'dev' into fix/dataset_assets_contributor_uploader

.github/workflows/cd-syft-dev.yml

@@ -1,9 +1,8 @@
 name: CD - Syft - Deploy `dev` to K8s
 
 on:
-  # TODO: Re-enable once merged & tested
-  # schedule:
-  #   - cron: "0 */3 * * *"
+  schedule:
+    - cron: "0 */3 * * *"
 
   workflow_dispatch:
     inputs:
@@ -15,6 +14,19 @@ jobs:
   deploy-syft-dev:
     runs-on: om-ci-16vcpu-ubuntu2204
     steps:
+      - name: Check for new changes
+        id: cache
+        if: github.event_name == 'schedule'
+        uses: actions/cache@v3
+        with:
+          path: scripts/commit_hash # we don't care about the file, just the key
+          key: dev-commit-${{ github.sha }}
+          lookup-only: true
+
+      - name: Quit if no new changes
+        if: github.event_name == 'schedule' && steps.cache.outputs.cache-hit == 'true'
+        run: exit 0
+
       - name: Permission to home directory
         run: |
           sudo chown -R $USER:$USER $HOME
@@ -39,7 +51,12 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to Azure container registry
+      - name: Login to Azure CLI
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }}}
+
+      - name: Login to Azure Container Registry
         uses: azure/docker-login@v1
         with:
           login-server: ${{ secrets.ACR_SERVER }}
@@ -51,7 +68,7 @@ jobs:
         shell: bash
         run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
 
-      - name: Build and push `grid-backend` image to DockerHub
+      - name: Build and push `grid-backend` image to registry
         uses: docker/build-push-action@v4
         with:
           context: ./packages
@@ -62,7 +79,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }}
 
-      - name: Build and push `grid-frontend` image to DockerHub
+      - name: Build and push `grid-frontend` image to registry
         uses: docker/build-push-action@v4
         with:
           context: ./packages/grid/frontend
@@ -74,7 +91,7 @@ jobs:
           target: grid-ui-development
 
       # TODO: Re-enable once we have Enclave up and running
-      # - name: Build and push `grid-enclave` image to DockerHub
+      # - name: Build and push `grid-enclave` image to registry
       #   uses: docker/build-push-action@v4
       #   with:
       #     context: ./packages
@@ -85,7 +102,7 @@ jobs:
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }}
 
-      - name: Build and push `grid-headscale` image to DockerHub
+      - name: Build and push `grid-headscale` image to registry
         uses: docker/build-push-action@v4
         with:
           context: ./packages/grid/vpn
@@ -95,7 +112,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }}
 
-      - name: Build and push `grid-tailscale` image to DockerHub
+      - name: Build and push `grid-tailscale` image to registry
         uses: docker/build-push-action@v4
         with:
           context: ./packages/grid/vpn
@@ -105,7 +122,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }}
 
-      - name: Build and push `grid-vpn-iptables` image to DockerHub
+      - name: Build and push `grid-vpn-iptables` image to registry
         uses: docker/build-push-action@v2
         with:
           context: ./packages/grid/vpn
@@ -150,3 +167,24 @@ jobs:
           add: "."
           push: "origin main"
           cwd: "./infrastructure/"
+
+      - name: Cleanup Azure Container Registry
+        uses: azure/CLI@v1
+        with:
+          # SKIP_LINES = latest version dev & dev-<sha> (2 lines) + keep "n" previous version (n lines) + 1
+          inlineScript: |
+            ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }}
+            REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv)
+
+            KEEP_PREV_VERSIONS=1
+            TAIL_FROM_LINE=$((2 + $KEEP_PREV_VERSIONS + 1))
+
+            for repo in $REPO_LIST
+            do
+                echo "Cleaning up '$repo'"
+                az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv | tail -n +$TAIL_FROM_LINE | xargs -r -I% az acr repository delete --name $ACR_REGISTRY_NAME --image $repo:% --yes
+            done
+
+      - name: Save Commit SHA
+        # only for cache to hit
+        run: echo "${{ github.sha }}" > scripts/commit_hash

.github/workflows/pr-tests-stack.yml

@@ -538,7 +538,7 @@ jobs:
           chmod 700 get_helm.sh
           ./get_helm.sh
 
-      - name: Run integration tests
+      - name: Run K8s & Helm integration tests
         if: steps.changes.outputs.stack == 'true'
         timeout-minutes: 60
         env:
@@ -556,7 +556,7 @@ jobs:
           curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
           chmod +x devspace
           devspace version
-          # tox -e stack.test.integration.k8s
+          tox -e stack.test.integration.k8s
           tox -e syft.build.helm
           tox -e syft.package.helm
           # tox -e syft.test.helm

.gitignore

@@ -70,3 +70,6 @@ js/node_modules/*
 
 #nohup
 nohup.out
+
+# tox syft.build.helm generated file
+out.txt

packages/grid/helm/helm.py

@@ -64,11 +64,9 @@ def replace_variables(d: Any) -> None:
     if "kubernetes.io/ingress.class" in d:
         d["kubernetes.io/ingress.class"] = "{{ .Values.ingress.ingressClass }}"
 
-    if "host" in d:
-        d["host"] = "{{ .Values.node.settings.hostname }}"
-
-    if "hosts" in d:
-        d["hosts"] = ["{{ .Values.node.settings.hostname }}"]
+    if "kind" in d and d["kind"] == "Ingress" and "spec" in d:
+        d["spec"]["tls"] = [{"hosts": ["{{ .Values.node.settings.hostname }}"]}]
+        d["spec"]["rules"][0]["host"] = "{{ .Values.node.settings.hostname }}"
 
 
 # parse whole tree
@@ -100,8 +98,8 @@ def main() -> None:
     )
     args = parser.parse_args()
     helm_dir = "helm"
-
     text = args.file.read()
+    file_count = 0
 
     # input_file = f"{helm_dir}/raw_manifests.yaml"
     # with open(input_file, "w") as f:
@@ -117,12 +115,12 @@ def main() -> None:
         )
         input_data = "---\n" + "\n".join(lines[first_index - 1 :])
     except StopIteration:
-        print("helm.py error: No line starting with 'apiVersion' found in the input.")
+        print("❌ Error: No line starting with 'apiVersion' found in the input.")
         print("------------------------------")
         print("Got input text:")
         print(text)
         print("------------------------------")
-        return
+        exit(1)
 
     helm_chart_template_dir = f"{helm_dir}/syft/templates"
 
@@ -168,6 +166,13 @@ def main() -> None:
 
             with open(new_file, "w") as f:
                 f.write(yaml_dump)  # add document separator
+                file_count += 1
+
+    if file_count > 0:
+        print(f"✅ Done: Generated {file_count} template files")
+    else:
+        print("❌ Failed: Generated zero files. Check input file for errors.")
+        exit(1)
 
 
 if __name__ == "__main__":

packages/grid/helm/manifests.yaml

@@ -1,21 +1,21 @@
 ---
 # Source: component-chart/templates/deployment.yaml
 # Create headless service for StatefulSet
+---
+
 ---
 # Source: component-chart/templates/ingress.yaml
 #
 
 
 ---
-
+# Source: component-chart/templates/deployment.yaml
+# Create headless service for StatefulSet
 ---
 # Source: component-chart/templates/ingress.yaml
 #
 
 
----
-# Source: component-chart/templates/deployment.yaml
-# Create headless service for StatefulSet
 ---
 
 ---
@@ -337,8 +337,7 @@ spec:
       port:
         number: 80
   rules:
-  - host: ""
-    http:
+  - http:
       paths:
       - backend:
           service:
@@ -347,9 +346,6 @@ spec:
               number: 80
         path: /
         pathType: Prefix
-  tls:
-  - hosts:
-    - ""
 
 ---
 # Source: component-chart/templates/service.yaml

packages/grid/helm/syft/values.yaml

@@ -22,7 +22,7 @@ db:
 
 node:
   settings:
-    hostname: "localhost"
+    hostname: ""
     nodeName: "mynode"
     nodeType: "domain"
     versionHash: "abc"

packages/grid/k8s/manifests/ingress.yaml

@@ -10,13 +10,8 @@ spec:
       name: proxy
       port:
         number: 80
-  tls:
-    - hosts:
-        - ""
-      # secretName: Add custom TLS secret here or default certificate is used
   rules:
-    - host: ""
-      http:
+    - http:
         paths:
           - backend:
               service:

packages/grid/vagrant/almalinux_9/arm_64/Vagrantfile

@@ -0,0 +1,38 @@
+Vagrant.configure("2") do |config|
+    config.vm.box = "almalinux/9.aarch64"
+    config.vm.hostname = "almalinux-9"
+
+    config.vm.provider "virtualbox" do |vb|
+        vb.memory = "4096"
+        vb.cpus = "2"
+    end
+
+    config.vm.provider "vmware_desktop" do |vd|
+        vd.memory = "4096"
+        vd.cpus = "2"
+    end
+
+    config.vm.box_check_update = false
+
+    config.vm.provision "shell", inline: <<-SHELL
+    #!/bin/bash
+    echo "Hello from the inline Bash script!"
+    # Flush existing rules
+    iptables-save > rules.v4.old
+    iptables -F
+
+    # Set the default policies to DROP
+    iptables -P INPUT DROP
+    iptables -P FORWARD DROP
+    iptables -P OUTPUT ACCEPT
+
+    # Allow incoming SSH traffic
+    iptables -A INPUT -p tcp --dport 22 -j ACCEPT
+
+    # Save the rules so they persist across reboots
+    iptables-save > rules.v4.new
+    pwd
+
+    SHELL
+
+  end

packages/grid/vagrant/almalinux_9/x86_64/Vagrantfile

@@ -0,0 +1,37 @@
+Vagrant.configure("2") do |config|
+    config.vm.box = "almalinux/9"
+    config.vm.hostname = "almalinux-9"
+
+    config.vm.provider "virtualbox" do |vb|
+        vb.memory = "4096"
+        vb.cpus = "2"
+    end
+
+    config.vm.provider "vmware_desktop" do |vd|
+        vd.memory = "4096"
+        vd.cpus = "2"
+    end
+    config.vm.box_check_update = false
+
+    config.vm.provision "shell", inline: <<-SHELL
+    #!/bin/bash
+    echo "Hello from the inline Bash script!"
+    # Flush existing rules
+    iptables-save > rules.v4.old
+    iptables -F
+
+    # Set the default policies to DROP
+    iptables -P INPUT DROP
+    iptables -P FORWARD DROP
+    iptables -P OUTPUT ACCEPT
+
+    # Allow incoming SSH traffic
+    iptables -A INPUT -p tcp --dport 22 -j ACCEPT
+
+    # Save the rules so they persist across reboots
+    iptables-save > rules.v4.new
+    pwd
+
+    SHELL
+
+  end

packages/grid/podman/vagrant/arm_64/Vagrantfile → packages/grid/vagrant/fedora_38/arm_64/Vagrantfile

@@ -11,7 +11,7 @@ Vagrant.configure("2") do |config|
         vd.memory = "4096"
         vd.cpus = "2"
     end
-    config.vm.synced_folder "../../podman-kube", "/home/vagrant/podman-kube" ,type: "rsync"
+    config.vm.synced_folder "../../../podman/podman-kube", "/home/vagrant/podman-kube" ,type: "rsync"
     config.vm.box_check_update = false
     config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1"
 

packages/grid/podman/vagrant/x86_64/Vagrantfile → packages/grid/vagrant/fedora_38/x86_64/Vagrantfile

@@ -1,6 +1,6 @@
 Vagrant.configure("2") do |config|
-    config.vm.box = "bento/fedora-38-x86_64"
-    config.vm.hostname = "fedora-38"
+    config.vm.box = "bento/fedora-38"
+    config.vm.hostname = "fedora-38-x86_64"
 
     config.vm.provider "virtualbox" do |vb|
         vb.memory = "4096"
@@ -11,7 +11,7 @@ Vagrant.configure("2") do |config|
         vd.memory = "4096"
         vd.cpus = "2"
     end
-    config.vm.synced_folder "../../podman-kube", "/home/vagrant/podman-kube" ,type: "rsync"
+    config.vm.synced_folder "../../../podman/podman-kube", "/home/vagrant/podman-kube" ,type: "rsync"
     config.vm.box_check_update = false
     config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1"