Merge branch 'dev' into dataset-index-2

OpenMined · Oct 27, 2023 · bbee1bb · bbee1bb
2 parents be2ee1a + 4dee56f
commit bbee1bb
Show file tree

Hide file tree

Showing 77 changed files with 602 additions and 2,471 deletions.
diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml
@@ -127,39 +127,6 @@ jobs:
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }}
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }}
 
-      - name: Build and push `grid-headscale` image to registry
-        uses: docker/build-push-action@v5
-        with:
-          context: ./packages/grid/vpn
-          file: ./packages/grid/vpn/headscale.dockerfile
-          push: true
-          tags: |
-            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev
-            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }}
-            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }}
-
-      - name: Build and push `grid-tailscale` image to registry
-        uses: docker/build-push-action@v5
-        with:
-          context: ./packages/grid/vpn
-          file: ./packages/grid/vpn/tailscale.dockerfile
-          push: true
-          tags: |
-            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev
-            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }}
-            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }}
-
-      - name: Build and push `grid-vpn-iptables` image to registry
-        uses: docker/build-push-action@v5
-        with:
-          context: ./packages/grid/vpn
-          file: ./packages/grid/vpn/iptables.dockerfile
-          push: true
-          tags: |
-            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev
-            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev-${{ github.sha }}
-            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }}
-
       - name: Build Helm Chart & Copy to infra
         if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
         shell: bash

diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml
@@ -260,66 +260,6 @@ jobs:
           cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }}
           cache-to: type=inline
 
-      - name: Docker meta for Grid-Headscale
-        id: meta_grid_headscale
-        uses: docker/metadata-action@v3
-        with:
-          images: openmined/grid-vpn-headscale
-          tags: |
-            type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
-            type=raw,value=beta
-
-      - name: Build and push `grid-headscale` image to DockerHub
-        uses: docker/build-push-action@v4
-        with:
-          context: ./packages/grid/vpn
-          file: ./packages/grid/vpn/headscale.dockerfile
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta_grid_headscale.outputs.tags }}
-          cache-from: type=registry,ref= ${{ steps.meta_grid_headscale.outputs.tags }}
-          cache-to: type=inline
-
-      - name: Docker meta for Grid-Tailscale
-        id: meta_grid_tailscale
-        uses: docker/metadata-action@v3
-        with:
-          images: openmined/grid-vpn-tailscale
-          tags: |
-            type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
-            type=raw,value=beta
-
-      - name: Build and push `grid-tailscale` image to DockerHub
-        uses: docker/build-push-action@v4
-        with:
-          context: ./packages/grid/vpn
-          file: ./packages/grid/vpn/tailscale.dockerfile
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta_grid_tailscale.outputs.tags }}
-          cache-from: type=registry,ref= ${{ steps.meta_grid_tailscale.outputs.tags }}
-          cache-to: type=inline
-
-      - name: Docker meta for grid-vpn-iptables
-        id: meta_grid_vpn_iptables
-        uses: docker/metadata-action@v3
-        with:
-          images: openmined/grid-vpn-iptables
-          tags: |
-            type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
-            type=raw,value=beta
-
-      - name: Build and push `grid-vpn-iptables` image to DockerHub
-        uses: docker/build-push-action@v2
-        with:
-          context: ./packages/grid/vpn
-          file: ./packages/grid/vpn/iptables.dockerfile
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta_grid_vpn_iptables.outputs.tags }}
-          cache-from: type=registry,ref=  ${{ steps.meta_grid_vpn_iptables.outputs.tags }}
-          cache-to: type=inline
-
       - name: Login to Docker-Helm
         run: |
           helm registry login registry-1.docker.io -u ${{ secrets.DOCKER_LOGIN }} --password ${{secrets.DOCKER_PASSWORD}}

diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml
@@ -21,9 +21,13 @@ jobs:
 
       # free 10GB of space
       - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       # Build the docker image for testing
       - name: Build a Docker image
@@ -64,9 +68,13 @@ jobs:
 
       # free 10GB of space
       - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       # Build the docker image for testing
       - name: Build a Docker image
@@ -169,154 +177,6 @@ jobs:
         with:
           sarif_file: snyk-code.sarif
 
-  scan-tailscale-trivy:
-    permissions:
-      contents: read # for actions/checkout to fetch code
-      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
-      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      # Build the docker image for testing
-      - name: Build a Docker image
-        shell: bash
-        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/vpn/tailscale.dockerfile packages/grid/vpn -t tailscale:${{ github.sha }} --no-cache
-        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.
-
-      - name: Run Trivy vulnerability scanner
-        continue-on-error: true
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "tailscale:${{ github.sha }}"
-          format: "template"
-          template: "@/contrib/sarif.tpl"
-          output: "trivy-results.sarif"
-          severity: "CRITICAL,HIGH"
-          timeout: "10m0s"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
-  scan-tailscale-snyk:
-    permissions:
-      contents: read # for actions/checkout to fetch code
-      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
-      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Snyk CLI to check for security issues
-        # Snyk can be used to break the build when it detects security issues.
-        # In this case we want to upload the SAST issues to GitHub Code Scanning
-        uses: snyk/actions/setup@master
-        env:
-          # This is where you will need to introduce the Snyk API token created with your Snyk account
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-
-      # Build the docker image for testing
-      - name: Build a Docker image
-        shell: bash
-        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/vpn/tailscale.dockerfile packages/grid/vpn -t tailscale:${{ github.sha }} --no-cache
-        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.
-
-      - name: Snyk auth
-        shell: bash
-        run: snyk config set api=$SNYK_TOKEN
-        env:
-          # This is where you will need to introduce the Snyk API token created with your Snyk account
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-
-      - name: Snyk Container test
-        continue-on-error: true
-        shell: bash
-        run: snyk container test tailscale:${{ github.sha }} --file=packages/grid/vpn/tailscale.dockerfile --sarif --sarif-file-output=snyk-code.sarif
-        env:
-          # This is where you will need to introduce the Snyk API token created with your Snyk account
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-
-        # Push the Snyk Code results into GitHub Code Scanning tab
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: snyk-code.sarif
-
-  scan-headscale-trivy:
-    permissions:
-      contents: read # for actions/checkout to fetch code
-      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
-      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      # Build the docker image for testing
-      - name: Build a Docker image
-        shell: bash
-        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/vpn/headscale.dockerfile packages/grid/vpn -t headscale:${{ github.sha }} --no-cache
-        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.
-
-      - name: Run Trivy vulnerability scanner
-        continue-on-error: true
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "headscale:${{ github.sha }}"
-          format: "template"
-          template: "@/contrib/sarif.tpl"
-          output: "trivy-results.sarif"
-          severity: "CRITICAL,HIGH"
-          timeout: "10m0s"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
-  scan-headscale-snyk:
-    permissions:
-      contents: read # for actions/checkout to fetch code
-      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
-      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Snyk CLI to check for security issues
-        # Snyk can be used to break the build when it detects security issues.
-        # In this case we want to upload the SAST issues to GitHub Code Scanning
-        uses: snyk/actions/setup@master
-        env:
-          # This is where you will need to introduce the Snyk API token created with your Snyk account
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-
-      # Build the docker image for testing
-      - name: Build a Docker image
-        shell: bash
-        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/vpn/headscale.dockerfile packages/grid/vpn -t headscale:${{ github.sha }} --no-cache
-        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.
-
-      - name: Snyk auth
-        shell: bash
-        run: snyk config set api=$SNYK_TOKEN
-        env:
-          # This is where you will need to introduce the Snyk API token created with your Snyk account
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-
-      - name: Snyk Container test
-        continue-on-error: true
-        shell: bash
-        run: snyk container test headscale:${{ github.sha }} --file=packages/grid/vpn/headscale.dockerfile --sarif --sarif-file-output=snyk-code.sarif
-        env:
-          # This is where you will need to introduce the Snyk API token created with your Snyk account
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-
-        # Push the Snyk Code results into GitHub Code Scanning tab
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: snyk-code.sarif
-
   scan-syft-requirements:
     permissions:
       contents: read # for actions/checkout to fetch code

diff --git a/.github/workflows/pr-tests-enclave.yml b/.github/workflows/pr-tests-enclave.yml
@@ -37,6 +37,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2

diff --git a/.github/workflows/pr-tests-frontend.yml b/.github/workflows/pr-tests-frontend.yml
@@ -114,6 +114,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2

diff --git a/.github/workflows/pr-tests-stack-arm64.yml b/.github/workflows/pr-tests-stack-arm64.yml
@@ -36,6 +36,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2

diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml
@@ -56,6 +56,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2
@@ -247,7 +250,7 @@ jobs:
       max-parallel: 99
       matrix:
         # os: [ubuntu-latest, macos-latest, windows-latest, windows]
-        os: [om-ci-16vcpu-ubuntu2204]
+        os: [ubuntu-latest]
         python-version: ["3.11"]
         notebook-paths: ["api/0.8"]
       fail-fast: false
@@ -268,6 +271,16 @@ jobs:
 
       - uses: actions/checkout@v3
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
       - name: Check for file changes
         uses: dorny/paths-filter@v2
         id: changes
@@ -459,7 +472,8 @@ jobs:
       max-parallel: 99
       matrix:
         # os: [ubuntu-latest, macos-latest, windows-latest, windows]
-        os: [om-ci-16vcpu-ubuntu2204]
+        # os: [om-ci-16vcpu-ubuntu2204]
+        os: [ubuntu-latest]
         python-version: ["3.11"]
         pytest-modules: ["frontend network"]
       fail-fast: false
@@ -485,6 +499,16 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
       - name: Get pip cache dir
         if: steps.changes.outputs.stack == 'true'
         id: pip-cache
@@ -544,6 +568,7 @@ jobs:
         env:
           HAGRID_ART: false
           PYTEST_MODULES: "${{ matrix.pytest-modules }}"
+          GITHUB_CI: true
         shell: bash
         run: |
           # install k3d