Merge branch 'dev' into madhava/deps

OpenMined · Sep 12, 2024 · b40d7b4 · b40d7b4
2 parents ed92f3b + 888aabb
commit b40d7b4
Show file tree

Hide file tree

Showing 100 changed files with 4,182 additions and 3,066 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.1-beta.11
+current_version = 0.9.2-beta.2
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.bumpversion_stable.cfg b/.bumpversion_stable.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.0
+current_version = 0.9.1
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml
@@ -60,9 +60,9 @@ jobs:
         if: steps.changes.outputs.stack == 'true'
         timeout-minutes: 60
         run: |
-          echo "Skipping pr image test"
+          tox -e backend.test.basecpu
         # run: |
-        #   tox -e backend.test.basecpu
+        #   echo "Skipping pr image test"
 
   pr-tests-syft-integration:
     strategy:
@@ -508,7 +508,7 @@ jobs:
           chmod 700 get_helm.sh
           ./get_helm.sh
 
-      - name: Run Notebooks Tests
+      - name: Run Notebooks Scenario Tests
         if: steps.changes.outputs.stack == 'true'
         timeout-minutes: 60
         env:
@@ -541,8 +541,8 @@ jobs:
         shell: bash
         run: |
           mkdir -p ./k8s-logs
-          kubectl describe all -A --context k3d-test-datasite-1 --namespace syft > ./k8s-logs/test-datasite-1-desc-${{ steps.date.outputs.date }}.txt
-          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-datasite-1 --namespace syft > ./k8s-logs/test-datasite-1-logs-${{ steps.date.outputs.date }}.txt
+          kubectl describe all -A --context k3d-bigquery-high --namespace syft > ./k8s-logs/bigquery-high-desc-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-bigquery-high --namespace syft > ./k8s-logs/bigquery-high-logs-${{ steps.date.outputs.date }}.txt
           ls -la ./k8s-logs
 
       - name: Upload logs to GitHub
@@ -620,3 +620,136 @@ jobs:
         if: steps.changes.outputs.syft == 'true'
         run: |
           tox -e migration.test
+
+  pr-tests-notebook-scenario-k8s-sync:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+      fail-fast: false
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+      - name: Permission to home directory
+        run: |
+          sudo chown -R $USER:$USER $HOME
+      - uses: actions/checkout@v4
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Add K3d Registry
+        run: |
+          sudo python ./scripts/patch_hosts.py --add-k3d-registry
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          large-packages: false
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+      - name: Install pip dependencies
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv==0.2.17 tox==4.16.0 tox-uv==1.9.0
+          uv --version
+      - name: Get uv cache dir
+        if: steps.changes.outputs.stack == 'true'
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
+      - name: Load github cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}
+          restore-keys: |
+            ${{ runner.os }}-uv-py${{ matrix.python-version }}
+      - name: Install kubectl
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # cleanup apt version
+          sudo apt remove kubectl || true
+          # install kubectl 1.27
+          curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl
+          chmod +x kubectl
+          sudo install kubectl /usr/local/bin;
+      - name: Install helm
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # install helm
+          curl  -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+          chmod 700 get_helm.sh
+          ./get_helm.sh
+      - name: Run Notebook Scenario Sync Tests
+        if: steps.changes.outputs.stack == 'true'
+        timeout-minutes: 60
+        env:
+          GITHUB_CI: true
+        shell: bash
+        run: |
+          K3D_VERSION=v5.6.3
+          DEVSPACE_VERSION=v6.3.12
+          # install k3d
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
+          mv k3d-linux-amd64 k3d
+          chmod +x k3d
+          export PATH=`pwd`:$PATH
+          k3d version
+          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
+          chmod +x devspace
+          devspace version
+          tox -e stack.test.notebook.scenario.k8s.sync
+      - name: Get current timestamp
+        id: date
+        if: failure()
+        shell: bash
+        run: echo "date=$(date +%s)" >> $GITHUB_OUTPUT
+
+      - name: Collect logs from k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          mkdir -p ./k8s-logs
+          kubectl describe all -A --context k3d-bigquery-low --namespace syft > ./k8s-logs/bigquery-low-desc-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-bigquery-low --namespace syft > ./k8s-logs/bigquery-low-logs-${{ steps.date.outputs.date }}.txt
+          kubectl describe all -A --context k3d-bigquery-high --namespace syft > ./k8s-logs/bigquery-high-desc-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-bigquery-high --namespace syft > ./k8s-logs/bigquery-high-logs-${{ steps.date.outputs.date }}.txt
+          ls -la ./k8s-logs
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@master
+        if: steps.changes.outputs.stack == 'true' && failure()
+        with:
+          name: k8s-logs-notebook-${{ matrix.os }}-${{ steps.date.outputs.date }}
+          path: ./k8s-logs/
+
+      - name: Cleanup k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          export PATH=`pwd`:$PATH
+          k3d cluster delete bigquery-high || true
+          k3d cluster delete bigquery-low || true
diff --git a/README.md b/README.md
@@ -41,7 +41,7 @@ Launch <a href="https://docs.openmined.org/en/latest/deployment/deployment-doc-1
 ```python
 import syft as sy
 
-sy.requires(">=0.9,<0.9.1")
+sy.requires(">=0.9.1,<0.9.2")
 
 server = sy.orchestra.launch(
     name="my-datasite",
@@ -70,7 +70,7 @@ Main way to use a Datasite is via our Syft client, in a Jupyter Notebook. Check
 ```python
 import syft as sy
 
-sy.requires(">=0.9,<0.9.1")
+sy.requires(">=0.9.1,<0.9.2")
 
 datasite_client = sy.login(
     port=8080,
@@ -138,12 +138,12 @@ For questions about PySyft, reach out via `#support` on <a href="https://slack.o
 
 **Latest Stable**
 
-- `0.9.0` (Stable) - <a href="https://docs.openmined.org/en/latest/index.html">Docs</a>
+- `0.9.1` (Stable) - <a href="https://docs.openmined.org/en/latest/index.html">Docs</a>
 - Install PySyft (Stable): `pip install -U syft`
 
 **Latest Beta**
 
-- `0.9.1` (Beta) - `dev` branch 👈🏽
+- `0.9.2` (Beta) - `dev` branch 👈🏽
 - Install PySyft (Beta): `pip install -U syft --pre`
 
 Find more about previous <a href="./releases.md">releases here</a>.

diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.9.1-beta.11"
+__version__ = "0.9.2-beta.2"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/notebooks/api/0.8/10-container-images.ipynb → notebooks/api/0.8/14-container-images.ipynb b/notebooks/api/0.8/10-container-images.ipynb → notebooks/api/0.8/14-container-images.ipynb
@@ -150,7 +150,7 @@
     "    if (bool(os.environ[\"DEV_MODE\"]) and running_as_container)\n",
     "    else sy.__version__\n",
     ")\n",
-    "syft_base_worker_tag = \"0.9.0-beta.5\""
+    "# syft_base_worker_tag = \"0.9.2-beta.1\""
    ]
   },
   {

diff --git a/...art-and-configure-server-and-admins.ipynb → ...art-and-configure-server-and-admins.ipynb b/...art-and-configure-server-and-admins.ipynb → ...art-and-configure-server-and-admins.ipynb
@@ -23,10 +23,21 @@
     "# stdlib\n",
     "from os import environ as env\n",
     "\n",
+    "# third party\n",
+    "# run email server\n",
+    "from helpers import get_email_server\n",
+    "\n",
     "# syft absolute\n",
     "import syft as sy"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Launch & login"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -49,23 +60,6 @@
     "env[\"DEFAULT_ROOT_PASSWORD\"] = ROOT_PASSWORD"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# third party\n",
-    "# run email server\n",
-    "from helpers import EmailServer\n",
-    "from helpers import SMTPTestServer\n",
-    "\n",
-    "email_server = EmailServer()\n",
-    "email_server.reset_emails()\n",
-    "smtp_server = SMTPTestServer(email_server)\n",
-    "smtp_server.start()"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -83,6 +77,15 @@
     ")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "email_server, smtp_server = get_email_server(reset=True)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -195,6 +198,13 @@
     "root_client.users.delete(new_user_id2)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Cleanup"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,