Merge branch 'dev' into hb-745-user-err-set-2

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.2-beta.32
+current_version = 0.8.2-beta.33
 tag = False
 tag_name = {new_version}
 commit = True

.github/workflows/cd-syft-dev.yml

@@ -1,9 +1,8 @@
 name: CD - Syft - Deploy `dev` to K8s
 
 on:
-  # TODO: Re-enable once merged & tested
-  # schedule:
-  #   - cron: "0 */3 * * *"
+  schedule:
+    - cron: "0 */3 * * *"
 
   workflow_dispatch:
     inputs:
@@ -15,6 +14,19 @@ jobs:
   deploy-syft-dev:
     runs-on: om-ci-16vcpu-ubuntu2204
     steps:
+      - name: Check for new changes
+        id: cache
+        if: github.event_name == 'schedule'
+        uses: actions/cache@v3
+        with:
+          path: scripts/commit_hash # we don't care about the file, just the key
+          key: dev-commit-${{ github.sha }}
+          lookup-only: true
+
+      - name: Quit if no new changes
+        if: github.event_name == 'schedule' && steps.cache.outputs.cache-hit == 'true'
+        run: exit 0
+
       - name: Permission to home directory
         run: |
           sudo chown -R $USER:$USER $HOME
@@ -39,7 +51,12 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to Azure container registry
+      - name: Login to Azure CLI
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }}}
+
+      - name: Login to Azure Container Registry
         uses: azure/docker-login@v1
         with:
           login-server: ${{ secrets.ACR_SERVER }}
@@ -51,7 +68,7 @@ jobs:
         shell: bash
         run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
 
-      - name: Build and push `grid-backend` image to DockerHub
+      - name: Build and push `grid-backend` image to registry
         uses: docker/build-push-action@v4
         with:
           context: ./packages
@@ -62,7 +79,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }}
 
-      - name: Build and push `grid-frontend` image to DockerHub
+      - name: Build and push `grid-frontend` image to registry
         uses: docker/build-push-action@v4
         with:
           context: ./packages/grid/frontend
@@ -74,7 +91,7 @@ jobs:
           target: grid-ui-development
 
       # TODO: Re-enable once we have Enclave up and running
-      # - name: Build and push `grid-enclave` image to DockerHub
+      # - name: Build and push `grid-enclave` image to registry
       #   uses: docker/build-push-action@v4
       #   with:
       #     context: ./packages
@@ -85,7 +102,7 @@ jobs:
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }}
 
-      - name: Build and push `grid-headscale` image to DockerHub
+      - name: Build and push `grid-headscale` image to registry
         uses: docker/build-push-action@v4
         with:
           context: ./packages/grid/vpn
@@ -95,7 +112,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }}
 
-      - name: Build and push `grid-tailscale` image to DockerHub
+      - name: Build and push `grid-tailscale` image to registry
         uses: docker/build-push-action@v4
         with:
           context: ./packages/grid/vpn
@@ -105,7 +122,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }}
 
-      - name: Build and push `grid-vpn-iptables` image to DockerHub
+      - name: Build and push `grid-vpn-iptables` image to registry
         uses: docker/build-push-action@v2
         with:
           context: ./packages/grid/vpn
@@ -150,3 +167,24 @@ jobs:
           add: "."
           push: "origin main"
           cwd: "./infrastructure/"
+
+      - name: Cleanup Azure Container Registry
+        uses: azure/CLI@v1
+        with:
+          # SKIP_LINES = latest version dev & dev-<sha> (2 lines) + keep "n" previous version (n lines) + 1
+          inlineScript: |
+            ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }}
+            REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv)
+
+            KEEP_PREV_VERSIONS=1
+            TAIL_FROM_LINE=$((2 + $KEEP_PREV_VERSIONS + 1))
+
+            for repo in $REPO_LIST
+            do
+                echo "Cleaning up '$repo'"
+                az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv | tail -n +$TAIL_FROM_LINE | xargs -r -I% az acr repository delete --name $ACR_REGISTRY_NAME --image $repo:% --yes
+            done
+
+      - name: Save Commit SHA
+        # only for cache to hit
+        run: echo "${{ github.sha }}" > scripts/commit_hash

.github/workflows/pr-tests-enclave.yml

@@ -31,6 +31,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Check for file changes
         uses: dorny/paths-filter@v2
         id: changes

.github/workflows/pr-tests-frontend.yml

@@ -108,6 +108,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Check for file changes
         uses: dorny/paths-filter@v2
         id: changes

.github/workflows/pr-tests-stack-arm64.yml

@@ -30,6 +30,13 @@ jobs:
 
       - uses: actions/checkout@v3
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Check for file changes
         uses: dorny/paths-filter@v2
         id: changes

.github/workflows/pr-tests-stack.yml

@@ -50,6 +50,13 @@ jobs:
 
       - uses: actions/checkout@v3
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Check for file changes
         uses: dorny/paths-filter@v2
         id: changes
@@ -531,7 +538,7 @@ jobs:
           chmod 700 get_helm.sh
           ./get_helm.sh
 
-      - name: Run integration tests
+      - name: Run K8s & Helm integration tests
         if: steps.changes.outputs.stack == 'true'
         timeout-minutes: 60
         env:
@@ -549,7 +556,7 @@ jobs:
           curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
           chmod +x devspace
           devspace version
-          # tox -e stack.test.integration.k8s
+          tox -e stack.test.integration.k8s
           tox -e syft.build.helm
           tox -e syft.package.helm
           # tox -e syft.test.helm

.gitignore

@@ -70,3 +70,6 @@ js/node_modules/*
 
 #nohup
 nohup.out
+
+# tox syft.build.helm generated file
+out.txt

VERSION

@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.2-beta.32"
+__version__ = "0.8.2-beta.33"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

packages/grid/VERSION

@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.2-beta.32"
+__version__ = "0.8.2-beta.33"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

packages/grid/devspace.yaml

@@ -15,13 +15,13 @@ pipelines:
     run: |-
       run_dependencies --all
       ensure_pull_secrets --all
-      build_images --all -t $(git rev-parse --short=6 HEAD) -t 0.8.2-beta.32 -t dev-latest
+      build_images --all -t $(git rev-parse --short=6 HEAD) -t 0.8.2-beta.33 -t dev-latest
       create_deployments --all
 
 vars:
   DEVSPACE_ENV_FILE: "default.env"
   CONTAINER_REGISTRY: "docker.io"
-  VERSION: "0.8.2-beta.32"
+  VERSION: "0.8.2-beta.33"
 
 # This is a list of `images` that DevSpace can build for this project
 # We recommend to skip image building during development (devspace dev) as much as possible
@@ -65,14 +65,18 @@ deployments:
       manifests:
         - "k8s/manifests/traefik-domain.yaml"
 
+  seaweedfs-config:
+    kubectl:
+      manifests:
+        - "k8s/manifests/seaweedfs.yaml"
+
   proxy:
     helm:
       upgradeArgs:
         - --dependency-update
       chart:
         name: component-chart
         repo: https://charts.devspace.sh
-        version: "0.9.0"
       values:
         containers:
           - image: "${DOCKER_IMAGE_TRAEFIK}:${TRAEFIK_VERSION}"
@@ -120,7 +124,6 @@ deployments:
       chart:
         name: component-chart
         repo: https://charts.devspace.sh
-        version: "0.9.0"
       values:
         containers:
           - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}:${VERSION}"
@@ -188,14 +191,14 @@ deployments:
           name: "backend"
           ports:
             - port: "${HTTP_PORT}"
+
   mongo:
     helm:
       upgradeArgs:
         - --dependency-update
       chart:
         name: component-chart
         repo: https://charts.devspace.sh
-        version: "0.9.0"
       values:
         containers:
           - image: "${MONGO_IMAGE}:${MONGO_VERSION}"
@@ -218,14 +221,63 @@ deployments:
           ports:
             - port: "27017"
 
+  seaweedfs:
+    helm:
+      upgradeArgs:
+        - --dependency-update
+      chart:
+        name: component-chart
+        repo: https://charts.devspace.sh
+      values:
+        containers:
+          - image: "${DOCKER_IMAGE_SEAWEEDFS}:${SEAWEEDFS_VERSION}"
+            command:
+              - sh
+              - /etc/seaweedfs/start.sh
+            env:
+              - name: S3_VOLUME_SIZE_MB
+                value: "${S3_VOLUME_SIZE_MB}"
+              - name: S3_ROOT_USER
+                value: "${S3_ROOT_USER}"
+              - name: S3_ROOT_PWD
+                value: "${S3_ROOT_PWD}"
+              - name: S3_PORT
+                value: "${S3_PORT}"
+            volumeMounts:
+              - containerPath: /etc/seaweedfs/filer.toml
+                volume:
+                  name: seaweedfs-config
+                  subPath: /filer.toml
+                  readOnly: false
+              - containerPath: /etc/seaweedfs/start.sh
+                volume:
+                  name: seaweedfs-config
+                  subPath: /start.sh
+                  readOnly: false
+              - containerPath: /data/blob
+                volume:
+                  name: seaweedfs-data
+                  subPath: /
+                  readOnly: false
+        volumes:
+          - name: seaweedfs-data
+            size: "5Gi"
+          - name: seaweedfs-config
+            configMap:
+              name: seaweedfs-config
+        service:
+          name: seaweedfs
+          ports:
+            - port: "8888" # filer
+            - port: "8333" # S3
+
   frontend:
     helm:
       upgradeArgs:
         - --dependency-update
       chart:
         name: component-chart
         repo: https://charts.devspace.sh
-        version: "0.9.0"
       values:
         containers:
           - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_FRONTEND}:${VERSION}"
@@ -248,6 +300,10 @@ dev:
     imageSelector: "${MONGO_IMAGE}:${MONGO_VERSION}"
     ports:
       - port: "27017"
+  seaweedfs:
+    imageSelector: "${DOCKER_IMAGE_SEAWEEDFS}:${SEAWEEDFS_VERSION}"
+    ports:
+      - port: "9333" # admin
   backend:
     imageSelector: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}"
     env: