Merge branch 'dev' of github.com:OpenMined/PySyft into helm

OpenMined · Oct 2, 2023 · 5c29f70 · 5c29f70
2 parents e6b2e58 + 8ecf66c
commit 5c29f70
Show file tree

Hide file tree

Showing 62 changed files with 1,136 additions and 225 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.2-beta.29
+current_version = 0.8.2-beta.33
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.github/workflows/cd-hagrid.yml b/.github/workflows/cd-hagrid.yml
@@ -46,7 +46,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN }}
+          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
       - name: Install checksumdir
         run: |
           pip install --upgrade checksumdir
@@ -85,13 +85,12 @@ jobs:
 
       - name: Commit changes
         if: ${{needs.hagrid-deploy.outputs.current_hash}} != ${{needs.hagrid-deploy.outputs.previous_hash}}
-        uses: EndBug/add-and-commit@v7
+        uses: EndBug/add-and-commit@v9
         with:
-          author_name: Madhava Jay
-          author_email: [email protected]
-          message: "bump version"
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
+          message: "[hagrid] bump version"
           add: "['./packages/hagrid/.bumpversion.cfg','./packages/hagrid/setup.py','./packages/hagrid/hagrid/version.py', './scripts/hagrid_hash', './packages/hagrid/hagrid/manifest_template.yml']"
-          pull_strategy: NO-PULL
 
       - name: Build and publish
         if: ${{needs.hagrid-deploy.outputs.current_hash}} != ${{needs.hagrid-deploy.outputs.previous_hash}}

diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml
@@ -0,0 +1,152 @@
+name: CD - Syft - Deploy `dev` to K8s
+
+on:
+  # TODO: Re-enable once merged & tested
+  # schedule:
+  #   - cron: "0 */3 * * *"
+
+  workflow_dispatch:
+    inputs:
+      none:
+        description: "Run Tests Manually"
+        required: false
+
+jobs:
+  deploy-syft-dev:
+    runs-on: om-ci-16vcpu-ubuntu2204
+    steps:
+      - name: Permission to home directory
+        run: |
+          sudo chown -R $USER:$USER $HOME
+
+      - uses: actions/checkout@v4
+
+      # Checkout Infra repo (nested)
+      - name: Checkout Infra Repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ secrets.INFRA_REPO }}
+          ref: "main"
+          token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
+          path: infrastructure
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Azure container registry
+        uses: azure/docker-login@v1
+        with:
+          login-server: ${{ secrets.ACR_SERVER }}
+          username: ${{ secrets.ACR_USERNAME }}
+          password: ${{ secrets.ACR_PASSWORD }}
+
+      - name: Set Grid package version
+        id: grid
+        shell: bash
+        run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
+
+      - name: Build and push `grid-backend` image to DockerHub
+        uses: docker/build-push-action@v4
+        with:
+          context: ./packages
+          file: ./packages/grid/backend/backend.dockerfile
+          push: true
+          target: backend
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }}
+
+      - name: Build and push `grid-frontend` image to DockerHub
+        uses: docker/build-push-action@v4
+        with:
+          context: ./packages/grid/frontend
+          file: ./packages/grid/frontend/frontend.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev-${{ github.sha }}
+          target: grid-ui-development
+
+      # TODO: Re-enable once we have Enclave up and running
+      # - name: Build and push `grid-enclave` image to DockerHub
+      #   uses: docker/build-push-action@v4
+      #   with:
+      #     context: ./packages
+      #     file: ./packages/grid/worker/worker.dockerfile
+      #     push: true
+      #     target: worker
+      #     tags: |
+      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev
+      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }}
+
+      - name: Build and push `grid-headscale` image to DockerHub
+        uses: docker/build-push-action@v4
+        with:
+          context: ./packages/grid/vpn
+          file: ./packages/grid/vpn/headscale.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }}
+
+      - name: Build and push `grid-tailscale` image to DockerHub
+        uses: docker/build-push-action@v4
+        with:
+          context: ./packages/grid/vpn
+          file: ./packages/grid/vpn/tailscale.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }}
+
+      - name: Build and push `grid-vpn-iptables` image to DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          context: ./packages/grid/vpn
+          file: ./packages/grid/vpn/iptables.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev-${{ github.sha }}
+
+      - name: Build Helm Chart & Copy to infra
+        shell: bash
+        run: |
+          K3D_VERSION=v5.6.0
+          DEVSPACE_VERSION=v6.3.3
+
+          # install k3d
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
+          mv k3d-linux-amd64 k3d
+          chmod +x k3d
+          export PATH=`pwd`:$PATH
+          k3d version
+
+          # Install devspace
+          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
+          chmod +x devspace
+          devspace version
+
+          export CONTAINER_REGISTRY=${{ secrets.ACR_SERVER }}
+          export VERSION=dev-${{github.sha}}
+          tox -e syft.build.helm
+
+          rm -rf infrastructure/gitops/environments/dev/
+          mkdir -p infrastructure/gitops/environments/dev/
+          cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/dev/
+
+      - name: Commit & push changes to Infra Repo
+        uses: EndBug/add-and-commit@v9
+        with:
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
+          message: "[env] Update dev helm charts"
+          add: "."
+          push: "origin main"
+          cwd: "./infrastructure/"
diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml
@@ -48,7 +48,8 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN }}
+          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
+
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
@@ -77,21 +78,20 @@ jobs:
           chmod +x k3d
           export PATH=`pwd`:$PATH
           k3d version
-          DEVSPACE_VERSION=v6.3.2
+          DEVSPACE_VERSION=v6.3.3
           curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
           chmod +x devspace
           devspace version
           tox -e syft.build.helm
           tox -e syft.package.helm
 
-      - name: Commit changes
-        uses: EndBug/add-and-commit@v7
+      - name: Commit changes to Syft
+        uses: EndBug/add-and-commit@v9
         with:
-          author_name: Madhava Jay
-          author_email: [email protected]
-          message: "bump version"
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
+          message: "[syft]bump version"
           add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml', 'packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml']"
-          pull: NO-PULL
 
       - name: Scheduled Build and Publish
         if: github.event_name == 'schedule'
@@ -131,6 +131,36 @@ jobs:
             echo "github_release_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
           fi
 
+      # Checkout Infra repo (nested)
+      - name: Checkout Infra Repo
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ secrets.INFRA_REPO }}
+          ref: "main"
+          token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
+          path: infrastructure
+
+      # This step will copy the generated K8s manifest files to the correct directory in Infra repo
+      - name: Copy files to Infra Repo
+        run: |
+          if $is_pre_release; then
+            rm -rf infrastructure/gitops/environments/pre-release/.
+            cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/pre_release/
+          else
+            rm -rf infrastructure/gitops/environments/stable/.
+            cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/stable/
+          fi
+
+      - name: Commit changes to Infra Repo
+        uses: EndBug/add-and-commit@v9
+        with:
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
+          message: "Update K8s Manifests from Syft Repo"
+          add: "."
+          push: "origin main"
+          cwd: "./infrastructure/"
+
       - name: Create SyftCLI Config assets
         run: |
           pip install pyyaml

diff --git a/.github/workflows/cd-syftcli.yml b/.github/workflows/cd-syftcli.yml
@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN  }}
+          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
 
       - name: Install checksumdir
         run: |
@@ -83,15 +83,14 @@ jobs:
         if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
         run: echo $(checksumdir packages/syftcli/syftcli -e version.py) > ./scripts/syftcli_hash
 
-      - name: Commit changes
+      - name: Commit changes to Syft CLI
         if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
-        uses: EndBug/add-and-commit@v7
+        uses: EndBug/add-and-commit@v9
         with:
-          author_name: Madhava Jay
-          author_email: [email protected]
-          message: "bump version"
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
+          message: "[syftcli]bump version"
           add: "['./packages/syftcli/.bumpversion.cfg','./packages/syftcli/setup.py','./packages/syftcli/syftcli/version.py', './scripts/syftcli_hash']"
-          pull_strategy: NO-PULL
 
       - name: Build and publish
         if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
@@ -114,8 +113,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN  }}
 
       - name: Set up Python
         uses: actions/setup-python@v3
@@ -152,8 +149,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN  }}
 
       - name: Deployed Version
         run: echo ${{needs.deploy-syft-cli.outputs.deployed_version}}

diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml
@@ -19,6 +19,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       # Build the docker image for testing
       - name: Build a Docker image
         run: DOCKER_BUILDKIT=1 docker build -f packages/grid/backend/backend.dockerfile packages -t backend:${{ github.sha }} --no-cache
@@ -56,6 +62,12 @@ jobs:
           # This is where you will need to introduce the Snyk API token created with your Snyk account
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       # Build the docker image for testing
       - name: Build a Docker image
         shell: bash
@@ -524,7 +536,7 @@ jobs:
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "seaweedfs:3.55"
+          image-ref: "chrislusf/seaweedfs:3.55"
           format: "cyclonedx"
           output: "seaweedfs-trivy-results.sbom.json"
           timeout: "10m0s"
@@ -541,7 +553,7 @@ jobs:
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "seaweedfs:3.55"
+          image-ref: "chrislusf/seaweedfs:3.55"
           format: "sarif"
           output: "trivy-results.sarif"
           timeout: "10m0s"

diff --git a/.github/workflows/nightlies.yml b/.github/workflows/nightlies.yml
@@ -42,7 +42,7 @@ jobs:
     uses: OpenMined/PySyft/.github/workflows/container-scan.yml@dev
     secrets: inherit
 
-  call-rhel-tests:
-    if: github.repository == 'OpenMined/PySyft' # don't run on forks
-    uses: OpenMined/PySyft/.github/workflows/rhel-tests.yml@dev
-    secrets: inherit
+  # call-rhel-tests:
+  #   if: github.repository == 'OpenMined/PySyft' # don't run on forks
+  #   uses: OpenMined/PySyft/.github/workflows/rhel-tests.yml@dev
+  #   secrets: inherit
diff --git a/.github/workflows/post-merge-cleanup-notebooks.yml b/.github/workflows/post-merge-cleanup-notebooks.yml
@@ -15,16 +15,18 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.SYFT_BUMP_TOKEN }}
+          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Commit changes
+
+      - name: Commit changes to remove notebooks
         uses: EndBug/add-and-commit@v9
         with:
-          author_name: Madhava Jay
-          author_email: [email protected]
+          author_name: ${{ secrets.OM_BOT_NAME }}
+          author_email: ${{ secrets.OM_BOT_EMAIL }}
           message: "cleanup notebooks"
           remove: "-r notebooks/Experimental/"
           commit: "-a"