Merge branch 'dev' into mongo-permission-model-tests

.github/workflows/cd-hagrid.yml

@@ -46,7 +46,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN }}
+          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
       - name: Install checksumdir
         run: |
           pip install --upgrade checksumdir
@@ -85,13 +85,12 @@ jobs:
 
       - name: Commit changes
         if: ${{needs.hagrid-deploy.outputs.current_hash}} != ${{needs.hagrid-deploy.outputs.previous_hash}}
-        uses: EndBug/add-and-commit@v7
+        uses: EndBug/add-and-commit@v9
         with:
-          author_name: Madhava Jay
-          author_email: [email protected]
-          message: "bump version"
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
+          message: "[hagrid] bump version"
           add: "['./packages/hagrid/.bumpversion.cfg','./packages/hagrid/setup.py','./packages/hagrid/hagrid/version.py', './scripts/hagrid_hash', './packages/hagrid/hagrid/manifest_template.yml']"
-          pull_strategy: NO-PULL
 
       - name: Build and publish
         if: ${{needs.hagrid-deploy.outputs.current_hash}} != ${{needs.hagrid-deploy.outputs.previous_hash}}

.github/workflows/cd-syft-dev.yml

@@ -0,0 +1,152 @@
+name: CD - Syft - Deploy `dev` to K8s
+
+on:
+  # TODO: Re-enable once merged & tested
+  # schedule:
+  #   - cron: "0 */3 * * *"
+
+  workflow_dispatch:
+    inputs:
+      none:
+        description: "Run Tests Manually"
+        required: false
+
+jobs:
+  deploy-syft-dev:
+    runs-on: om-ci-16vcpu-ubuntu2204
+    steps:
+      - name: Permission to home directory
+        run: |
+          sudo chown -R $USER:$USER $HOME
+
+      - uses: actions/checkout@v4
+
+      # Checkout Infra repo (nested)
+      - name: Checkout Infra Repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ secrets.INFRA_REPO }}
+          ref: "main"
+          token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
+          path: infrastructure
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Azure container registry
+        uses: azure/docker-login@v1
+        with:
+          login-server: ${{ secrets.ACR_SERVER }}
+          username: ${{ secrets.ACR_USERNAME }}
+          password: ${{ secrets.ACR_PASSWORD }}
+
+      - name: Set Grid package version
+        id: grid
+        shell: bash
+        run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
+
+      - name: Build and push `grid-backend` image to DockerHub
+        uses: docker/build-push-action@v4
+        with:
+          context: ./packages
+          file: ./packages/grid/backend/backend.dockerfile
+          push: true
+          target: backend
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }}
+
+      - name: Build and push `grid-frontend` image to DockerHub
+        uses: docker/build-push-action@v4
+        with:
+          context: ./packages/grid/frontend
+          file: ./packages/grid/frontend/frontend.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev-${{ github.sha }}
+          target: grid-ui-development
+
+      # TODO: Re-enable once we have Enclave up and running
+      # - name: Build and push `grid-enclave` image to DockerHub
+      #   uses: docker/build-push-action@v4
+      #   with:
+      #     context: ./packages
+      #     file: ./packages/grid/worker/worker.dockerfile
+      #     push: true
+      #     target: worker
+      #     tags: |
+      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev
+      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }}
+
+      - name: Build and push `grid-headscale` image to DockerHub
+        uses: docker/build-push-action@v4
+        with:
+          context: ./packages/grid/vpn
+          file: ./packages/grid/vpn/headscale.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }}
+
+      - name: Build and push `grid-tailscale` image to DockerHub
+        uses: docker/build-push-action@v4
+        with:
+          context: ./packages/grid/vpn
+          file: ./packages/grid/vpn/tailscale.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }}
+
+      - name: Build and push `grid-vpn-iptables` image to DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          context: ./packages/grid/vpn
+          file: ./packages/grid/vpn/iptables.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev-${{ github.sha }}
+
+      - name: Build Helm Chart & Copy to infra
+        shell: bash
+        run: |
+          K3D_VERSION=v5.6.0
+          DEVSPACE_VERSION=v6.3.3
+
+          # install k3d
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
+          mv k3d-linux-amd64 k3d
+          chmod +x k3d
+          export PATH=`pwd`:$PATH
+          k3d version
+
+          # Install devspace
+          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
+          chmod +x devspace
+          devspace version
+
+          export CONTAINER_REGISTRY=${{ secrets.ACR_SERVER }}
+          export VERSION=dev-${{github.sha}}
+          tox -e syft.build.helm
+
+          rm -rf infrastructure/gitops/environments/dev/
+          mkdir -p infrastructure/gitops/environments/dev/
+          cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/dev/
+
+      - name: Commit & push changes to Infra Repo
+        uses: EndBug/add-and-commit@v9
+        with:
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
+          message: "[env] Update dev helm charts"
+          add: "."
+          push: "origin main"
+          cwd: "./infrastructure/"

.github/workflows/cd-syft.yml

@@ -48,7 +48,7 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.OM_BOT_TOKEN }}
+          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
 
       - name: Set up Python
         uses: actions/setup-python@v3
@@ -90,7 +90,7 @@ jobs:
         with:
           author_name: ${{ secrets.OM_BOT_NAME }}
           author_email: ${{ secrets.OM_BOT_EMAIL }}
-          message: "bump version"
+          message: "[syft]bump version"
           add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml', 'packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml']"
 
       - name: Scheduled Build and Publish
@@ -137,7 +137,7 @@ jobs:
         with:
           repository: ${{ secrets.INFRA_REPO }}
           ref: "main"
-          token: ${{ secrets.OM_BOT_TOKEN }}
+          token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
           path: infrastructure
 
       # This step will copy the generated K8s manifest files to the correct directory in Infra repo
@@ -157,7 +157,7 @@ jobs:
           author_name: ${{ secrets.OM_BOT_NAME }}
           author_email: ${{ secrets.OM_BOT_EMAIL }}
           message: "Update K8s Manifests from Syft Repo"
-          add: "*"
+          add: "."
           push: "origin main"
           cwd: "./infrastructure/"
 

.github/workflows/cd-syftcli.yml

@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN  }}
+          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
 
       - name: Install checksumdir
         run: |
@@ -83,15 +83,14 @@ jobs:
         if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
         run: echo $(checksumdir packages/syftcli/syftcli -e version.py) > ./scripts/syftcli_hash
 
-      - name: Commit changes
+      - name: Commit changes to Syft CLI
         if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
-        uses: EndBug/add-and-commit@v7
+        uses: EndBug/add-and-commit@v9
         with:
-          author_name: Madhava Jay
-          author_email: [email protected]
-          message: "bump version"
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
+          message: "[syftcli]bump version"
           add: "['./packages/syftcli/.bumpversion.cfg','./packages/syftcli/setup.py','./packages/syftcli/syftcli/version.py', './scripts/syftcli_hash']"
-          pull_strategy: NO-PULL
 
       - name: Build and publish
         if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
@@ -114,8 +113,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN  }}
 
       - name: Set up Python
         uses: actions/setup-python@v3
@@ -152,8 +149,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN  }}
 
       - name: Deployed Version
         run: echo ${{needs.deploy-syft-cli.outputs.deployed_version}}

.github/workflows/container-scan.yml

@@ -19,6 +19,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       # Build the docker image for testing
       - name: Build a Docker image
         run: DOCKER_BUILDKIT=1 docker build -f packages/grid/backend/backend.dockerfile packages -t backend:${{ github.sha }} --no-cache
@@ -56,6 +62,12 @@ jobs:
           # This is where you will need to introduce the Snyk API token created with your Snyk account
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       # Build the docker image for testing
       - name: Build a Docker image
         shell: bash
@@ -524,7 +536,7 @@ jobs:
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "seaweedfs:3.55"
+          image-ref: "chrislusf/seaweedfs:3.55"
           format: "cyclonedx"
           output: "seaweedfs-trivy-results.sbom.json"
           timeout: "10m0s"
@@ -541,7 +553,7 @@ jobs:
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "seaweedfs:3.55"
+          image-ref: "chrislusf/seaweedfs:3.55"
           format: "sarif"
           output: "trivy-results.sarif"
           timeout: "10m0s"

.github/workflows/post-merge-cleanup-notebooks.yml

@@ -15,16 +15,18 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN }}
+          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Commit changes
+
+      - name: Commit changes to remove notebooks
         uses: EndBug/add-and-commit@v9
         with:
-          author_name: Madhava Jay
-          author_email: [email protected]
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
           message: "cleanup notebooks"
           remove: "-r notebooks/Experimental/"
           commit: "-a"

.github/workflows/pr-tests-frontend.yml

@@ -148,7 +148,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p ~/.docker/cli-plugins
-          DOCKER_COMPOSE_VERSION=v2.16.0
+          DOCKER_COMPOSE_VERSION=v2.21.0
           curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
           chmod +x ~/.docker/cli-plugins/docker-compose
 

.github/workflows/pr-tests-stack-arm64.yml

@@ -70,7 +70,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p ~/.docker/cli-plugins
-          DOCKER_COMPOSE_VERSION=v2.16.0
+          DOCKER_COMPOSE_VERSION=v2.21.0
           curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
           chmod +x ~/.docker/cli-plugins/docker-compose
 

.github/workflows/pr-tests-stack-public.yml

@@ -111,7 +111,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p ~/.docker/cli-plugins
-          DOCKER_COMPOSE_VERSION=v2.16.0
+          DOCKER_COMPOSE_VERSION=v2.21.0
           curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
           chmod +x ~/.docker/cli-plugins/docker-compose
 

.github/workflows/pr-tests-stack.yml

@@ -123,7 +123,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p ~/.docker/cli-plugins
-          DOCKER_COMPOSE_VERSION=v2.16.0
+          DOCKER_COMPOSE_VERSION=v2.21.0
           curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
           chmod +x ~/.docker/cli-plugins/docker-compose
 
@@ -334,7 +334,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p ~/.docker/cli-plugins
-          DOCKER_COMPOSE_VERSION=v2.16.0
+          DOCKER_COMPOSE_VERSION=v2.21.0
           curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
           chmod +x ~/.docker/cli-plugins/docker-compose
 

.github/workflows/pr-tests-syft.yml

@@ -208,6 +208,14 @@ jobs:
       #   run: |
       #     sudo chown -R $USER:$USER $HOME
       - uses: actions/checkout@v3
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Check for file changes
         uses: dorny/paths-filter@v2
         id: changes
@@ -253,7 +261,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p ~/.docker/cli-plugins
-          DOCKER_COMPOSE_VERSION=v2.16.0
+          DOCKER_COMPOSE_VERSION=v2.21.0
           curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
           chmod +x ~/.docker/cli-plugins/docker-compose
           docker compose version