Merge branch 'dev' into yash/fix-cd-syft-dev

OpenMined · Feb 15, 2024 · c2436aa · c2436aa
2 parents 543af6a + a0f011e
commit c2436aa
Show file tree

Hide file tree

Showing 49 changed files with 623 additions and 283 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.4-beta.21
+current_version = 0.8.4-beta.22
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.github/workflows/cd-hagrid.yml b/.github/workflows/cd-hagrid.yml
@@ -11,6 +11,12 @@ on:
         required: false
         default: "false"
 
+# Prevents concurrent runs of the same workflow
+# while the previous run is still in progress
+concurrency:
+  group: "CD - Hagrid"
+  cancel-in-progress: false
+
 jobs:
   call-pr-tests-linting:
     if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks

diff --git a/.github/workflows/cd-post-release-tests.yml b/.github/workflows/cd-post-release-tests.yml
@@ -0,0 +1,55 @@
+name: CD - Post Release Tests
+
+on:
+  workflow_call:
+
+  workflow_dispatch:
+    inputs:
+      syft_version:
+        description: "Syft version to test"
+        required: true
+        type: string
+
+jobs:
+  syft-install-check:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.11", "3.10", "3.9"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install Syft
+        run: |
+          pip install syft==${{ github.event.inputs.syft_version }}
+
+      - name: Check Syft version
+        run: |
+          python -c "import syft; print(syft.__version__)"
diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
@@ -106,3 +106,65 @@ jobs:
         if: steps.changes.outputs.hagrid == 'true'
         run: |
           twine check dist/*.whl
+
+  pr-tests-syft-hagrid-comptability:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+        syft-version: ["0.8.2", "0.8.2b6", "0.8.3"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.hagrid == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        if: steps.changes.outputs.hagrid == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.hagrid == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('packages/syft/setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('packages/syft/setup.cfg') }}
+
+      # https://github.com/google/jax/issues/17693
+      # pinning ml-dtypes due to jax version==0.4.10
+      - name: Install Syft ${{ matrix.syft-version }}
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          pip install ml-dtypes==0.2.0
+          pip install syft==${{ matrix.syft-version }}
+          pip install .
+
+      - name: Run Orchestra Command
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          python -c "import syft as sy; domain1 = sy.orchestra.launch(name='test-domain-1', dev_mode=True, reset=True)"
+          python -c "import syft as sy; domain2 = sy.orchestra.launch(name='test-domain-2',dev_mode=False, reset=True)"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -172,7 +172,7 @@ repos:
       - id: mypy
         name: "mypy: syft"
         always_run: true
-        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py"
+        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py|^packages/syft/src/syft/service/warnings.py|^packages/syft/src/syft/service/dataset|^packages/syft/src/syft/service/worker"
         #files: "^packages/syft/src/syft/serde"
         args: [
             "--follow-imports=skip",

diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.4-beta.21"
+__version__ = "0.8.4-beta.22"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb
@@ -163,11 +163,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "default_worker_pool = domain_client.api.services.worker_pool.scale(\n",
+    "default_pool_scale_res = domain_client.api.services.worker_pool.scale(\n",
     "    number=1, pool_name=\"default-pool\"\n",
     ")\n",
-    "assert not isinstance(result, sy.SyftError), str(result)\n",
-    "default_worker_pool"
+    "assert not isinstance(default_pool_scale_res, sy.SyftError), str(default_pool_scale_res)\n",
+    "default_pool_scale_res"
    ]
   },
   {
@@ -850,6 +850,31 @@
     "assert result_matches.all()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6b5f63c2-028a-4b48-a5f9-392ac89440ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Scale Down the workers\n",
+    "custom_pool_scale_res = domain_client.api.services.worker_pool.scale(\n",
+    "    number=1, pool_name=worker_pool_name\n",
+    ")\n",
+    "assert not isinstance(custom_pool_scale_res, sy.SyftError), str(custom_pool_scale_res)\n",
+    "custom_pool_scale_res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3325165b-525f-4ffd-add5-e0c93d235723",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert len(domain_client.worker_pools[worker_pool_name].worker_list) == 1"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "f20a29df-2e63-484f-8b67-d6a397722e66",
@@ -1080,6 +1105,31 @@
     "assert len(worker_pool_list) == 3"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18ddb1e7-8d8b-480c-b6a4-e4c79d27bcf1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Scale Down the workers\n",
+    "opendp_pool_scale_res = domain_client.api.services.worker_pool.scale(\n",
+    "    number=1, pool_name=pool_name_opendp\n",
+    ")\n",
+    "assert not isinstance(opendp_pool_scale_res, sy.SyftError), str(opendp_pool_scale_res)\n",
+    "opendp_pool_scale_res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "83b3ec7b-3fbe-429d-bd1e-5e9afa223c3c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert len(domain_client.worker_pools[pool_name_opendp].worker_list) == 1"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "6e671e1e",
@@ -1238,6 +1288,33 @@
     "assert domain_client.worker_pools[pool_name_recordlinkage]\n",
     "assert len(domain_client.worker_pools[pool_name_recordlinkage].worker_list) == 2"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0cec28e8-784e-4a8d-91f9-f2481a967008",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Scale down the workers\n",
+    "recordlinkage_pool_scale_res = domain_client.api.services.worker_pool.scale(\n",
+    "    number=1, pool_name=pool_name_recordlinkage\n",
+    ")\n",
+    "assert not isinstance(recordlinkage_pool_scale_res, sy.SyftError), str(\n",
+    "    recordlinkage_pool_scale_res\n",
+    ")\n",
+    "recordlinkage_pool_scale_res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a43cf8cf-b8ca-4df4-aec9-6651d0a2fcda",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert len(domain_client.worker_pools[pool_name_recordlinkage].worker_list) == 1"
+   ]
   }
  ],
  "metadata": {
@@ -1256,7 +1333,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,

diff --git a/packages/grid/VERSION b/packages/grid/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.4-beta.21"
+__version__ = "0.8.4-beta.22"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/packages/grid/backend/worker_cpu.dockerfile b/packages/grid/backend/worker_cpu.dockerfile
@@ -9,7 +9,7 @@
 # Later we'd want to uninstall old python, and then install a new python runtime...
 # ... but pre-built syft deps may break!
 
-ARG SYFT_VERSION_TAG="0.8.4-beta.21"
+ARG SYFT_VERSION_TAG="0.8.4-beta.22"
 FROM openmined/grid-backend:${SYFT_VERSION_TAG}
 
 ARG PYTHON_VERSION="3.11"

diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml
@@ -25,7 +25,7 @@ vars:
   DEVSPACE_ENV_FILE: "default.env"
   CONTAINER_REGISTRY: "docker.io"
   NODE_NAME: "mynode"
-  VERSION: "0.8.4-beta.21"
+  VERSION: "0.8.4-beta.22"
 
 # This is a list of `images` that DevSpace can build for this project
 # We recommend to skip image building during development (devspace dev) as much as possible

diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "pygrid-ui",
-  "version": "0.8.4-beta.21",
+  "version": "0.8.4-beta.22",
   "private": true,
   "scripts": {
     "dev": "pnpm i && vite dev --host --port 80",