Merge pull request #8514 from OpenMined/dev

Update 0.8.4

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.4-beta.22
+current_version = 0.8.4
 tag = False
 tag_name = {new_version}
 commit = True
@@ -28,14 +28,14 @@ first_value = 1
 
 [bumpversion:file:VERSION]
 
-[bumpversion:file:packages/grid/devspace.yaml]
-
 [bumpversion:file:packages/syft/setup.cfg]
 
 [bumpversion:file:packages/syft/src/syft/__init__.py]
 
 [bumpversion:file:packages/syft/src/syft/VERSION]
 
+[bumpversion:file:packages/grid/devspace.yaml]
+
 [bumpversion:file:packages/grid/VERSION]
 
 [bumpversion:file:packages/grid/backend/worker_cpu.dockerfile]
@@ -44,14 +44,14 @@ first_value = 1
 
 [bumpversion:file:packages/grid/helm/syft/Chart.yaml]
 
-[bumpversion:file:packages/hagrid/hagrid/manifest_template.yml]
-
-[bumpversion:file:packages/hagrid/hagrid/deps.py]
-
 [bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube.yaml]
 
 [bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube-config.yaml]
 
-[bumpversion:file:packages/syftcli/manifest.yml]
-
 [bumpversion:file:packages/grid/helm/syft/values.yaml]
+
+[bumpversion:file:packages/hagrid/hagrid/manifest_template.yml]
+
+[bumpversion:file:packages/hagrid/hagrid/deps.py]
+
+[bumpversion:file:packages/syftcli/manifest.yml]

.bumpversion_stable.cfg

@@ -0,0 +1,19 @@
+[bumpversion]
+current_version = 0.8.4
+tag = False
+tag_name = {new_version}
+commit = True
+parse =
+	(?P<major>\d+)
+	\.
+	(?P<minor>\d+)
+	\.
+	(?P<patch>\d+)
+serialize =
+	{major}.{minor}.{patch}
+
+[bumpversion:file:packages/syft/src/syft/stable_version.py]
+
+[bumpversion:file:packages/hagrid/hagrid/stable_version.py]
+
+[bumpversion:file:packages/hagrid/hagrid/cache.py]

.github/workflows/cd-post-release-tests.yml

@@ -1,25 +1,134 @@
 name: CD - Post Release Tests
 
 on:
-  workflow_call:
-
   workflow_dispatch:
     inputs:
       syft_version:
         description: "Syft version to test"
         required: true
         type: string
 
+  workflow_call:
+    inputs:
+      syft_version:
+        description: "Syft version to test"
+        required: true
+        type: string
+
 jobs:
+  notebook-test-hagrid:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install Hagrid and tox
+        run: |
+          pip install -U hagrid
+          pip install tox
+
+      - name: Hagrid Version
+        run: |
+          hagrid version
+
+      - name: Remove existing containers
+        continue-on-error: true
+        shell: bash
+        run: |
+          docker rm $(docker ps -aq) --force || true
+          docker volume prune -f || true
+          docker buildx use default || true
+
+      - name: Launch Domain
+        run: |
+          hagrid launch test-domain-1 to docker:8081 --tag=${{ inputs.syft_version }} --low-side
+
+      - name: Run tests
+        env:
+          NODE_PORT: "8081"
+          SYFT_VERSION: ${{ inputs.syft_version }}
+          EXCLUDE_NOTEBOOKS: "not 11-container-images-k8s.ipynb"
+        run: |
+          tox -e e2e.test.notebook
+
+      #Run log collector python script
+      - name: Run log collector
+        timeout-minutes: 5
+        if: failure()
+        shell: bash
+        run: |
+          python ./scripts/container_log_collector.py
+
+      # Get Job name and url
+      - name: Get job name and url
+        id: job_name
+        if: failure()
+        shell: bash
+        run: |
+          echo "job_name=$(echo ${{ github.job }})" >> $GITHUB_OUTPUT
+          echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@master
+        if: failure()
+        with:
+          name: ${{ matrix.os }}-${{ steps.job_name.outputs.job_name }}-logs-${{ steps.job_name.outputs.date }}
+          path: ./logs/${{ steps.job_name.outputs.job_name}}/
+
   syft-install-check:
     strategy:
       max-parallel: 99
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
+        # Re-enable macos-14 when it's available when we remove python 3.9 from syft
+        # os: [ubuntu-latest, macos-latest, windows-latest, macos-14]
         python-version: ["3.11", "3.10", "3.9"]
-
     runs-on: ${{ matrix.os }}
     steps:
+      - name: System Architecture
+        run: |
+          echo "System Architecture: $(uname -m)"
+          echo "System Version: $(uname -a)"
+
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
@@ -48,8 +157,64 @@ jobs:
 
       - name: Install Syft
         run: |
-          pip install syft==${{ github.event.inputs.syft_version }}
+          pip install syft==${{ inputs.syft_version }}
 
       - name: Check Syft version
         run: |
           python -c "import syft; print(syft.__version__)"
+
+  notebook-test-k8s-k3d:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install K3d
+        run: |
+          K3D_VERSION=v5.6.0
+          wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=$K3D_VERSION bash
+
+      - name: Get pip cache dir
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install tox
+        run: |
+          pip install tox
+
+      - name: Run K8s tests
+        env:
+          SYFT_VERSION: ${{ inputs.syft_version }}
+        run: |
+          tox -e syft.test.helm

.github/workflows/cd-syft-dev.yml

@@ -70,20 +70,10 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Install Azure CLI
-        run: |
-          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-          az version
-
-      - name: Login to Azure CLI
-        uses: azure/login@v1
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }}
-
       - name: Login to Azure Container Registry
-        uses: azure/docker-login@v1
+        uses: docker/login-action@v3
         with:
-          login-server: ${{ secrets.ACR_SERVER }}
+          registry: ${{ secrets.ACR_SERVER }}
           username: ${{ secrets.ACR_USERNAME }}
           password: ${{ secrets.ACR_PASSWORD }}
 
@@ -178,29 +168,3 @@ jobs:
           add: "."
           push: "origin main"
           cwd: "./infrastructure/"
-
-      - name: Cleanup Azure Container Registry
-        run: |
-          ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }}
-
-          echo ">> Fetching repo list.."
-          REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv)
-
-          KEEP_PREV_VERSIONS=5
-          TAIL_FROM_LINE=$(($KEEP_PREV_VERSIONS + 1))
-
-          for repo in $REPO_LIST
-          do
-              echo "Cleaning up '$repo'"
-              az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv \
-                | grep dev- \
-                | tail -n +$TAIL_FROM_LINE \
-                | xargs -r -I% az acr repository untag --name $ACR_REGISTRY_NAME --image $repo:%
-          done
-
-      - name: Logout and cleanup Azure account
-        if: always()
-        run: |
-          az logout
-          az cache purge
-          az account clear

.github/workflows/cd-syft.yml

@@ -312,6 +312,10 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write # For tag and release notes.
+
+    outputs:
+      syft_version: ${{ steps.release_checks.outputs.syft_version }}
+
     steps:
       - name: Permission to home directory
         run: |
@@ -354,6 +358,7 @@ jobs:
         run: |
           echo "github_release_version=$(python packages/grid/VERSION |  sed 's/-beta./b/')" >> $GITHUB_OUTPUT
           echo "future_stable_version=$(python packages/grid/VERSION |   sed 's/-beta.*//')" >> $GITHUB_OUTPUT
+          echo "syft_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
           if [[ ${{ needs.merge-docker-images.outputs.release_tag}} == "beta" ]]; then
             echo "is_beta_release=true" >> $GITHUB_OUTPUT
           else
@@ -419,7 +424,7 @@ jobs:
           author_name: ${{ secrets.OM_BOT_NAME }}
           author_email: ${{ secrets.OM_BOT_EMAIL }}
           message: "[syft] bump protocol version"
-          add: "['packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/']"
+          add: "['packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/','packages/syft/PYPI.md','packages/grid/helm/repo']"
 
       - name: Scheduled Build and Publish
         if: github.event_name == 'schedule'
@@ -514,3 +519,17 @@ jobs:
           add: "helm/"
           push: "origin gh-pages"
           cwd: "./ghpages/"
+
+      # Since the post-release tests are dependent on the release being available on PyPI
+      # we need to wait for PyPI to update before running the post-release tests
+      - name: Wait for PyPI to update
+        run: |
+          sleep 60
+
+  # Can we remove the always flag from the below job?
+  call-cd-post-release-tests:
+    needs: [deploy-syft]
+    if: always() && github.repository == 'OpenMined/PySyft' && needs.deploy-syft.result == 'success' # don't run on forks
+    uses: OpenMined/PySyft/.github/workflows/cd-post-release-tests.yml@dev
+    with:
+      syft_version: ${{ needs.deploy-syft.outputs.syft_version }}