Merge branch 'dev' into padawan-contrib-3

.github/workflows/conda-install.yml

@@ -0,0 +1,57 @@
+name: Conda Install - PySyft
+
+on:
+  workflow_call:
+
+  workflow_dispatch:
+    inputs:
+      none:
+        description: "Run Version Tests Manually"
+        required: false
+
+jobs:
+  constall-install-syft:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.12"]
+      fail-fast: false
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          activate-environment: syft_conda_env
+          python-version: ${{ matrix.python-version }}
+      - name: Install syft (Windows)
+        if: matrix.os == 'windows-latest'
+        shell: pwsh
+        run: |
+          python -m pip install ./packages/syft
+          $expectedVersion = python packages/grid/VERSION
+          $syftVersion = python -c 'import syft; print(syft.__version__)'
+          # Compare the versions
+          if ($expectedVersion -ne $syftVersion) {
+            Write-Output "Expected version: $expectedVersion"
+            Write-Output "Actual version: $syftVersion"
+            Write-Output "Version mismatch."
+            exit 1
+          }
+      - name: Install syft (MacOS or Linux)
+        if: matrix.os != 'windows-latest'
+        shell: bash -el {0}
+        run: |
+          python -m pip install ./packages/syft
+          EXPECTED_VERSION=$(python packages/grid/VERSION)
+          SYFT_VERSION=$(python -c 'import syft; print(syft.__version__)')
+          # Compare the versions
+          if [ "$EXPECTED_VERSION" != "$SYFT_VERSION" ]; then
+            echo "Expected version: $EXPECTED_VERSION"
+            echo "Actual version: $SYFT_VERSION"
+            echo "Version mismatch."
+            exit 1
+          fi

.github/workflows/nightlies.yml

@@ -27,3 +27,8 @@ jobs:
     if: github.repository == 'OpenMined/PySyft' # don't run on forks
     uses: OpenMined/PySyft/.github/workflows/container-scan.yml@dev
     secrets: inherit
+
+  call-conda-install:
+    if: github.repository == 'OpenMined/PySyft' # don't run on forks
+    uses: OpenMined/PySyft/.github/workflows/conda-install.yml@dev
+    secrets: inherit

.github/workflows/pr-tests-stack.yml

@@ -415,3 +415,64 @@ jobs:
           k3d cluster delete test-gateway-1 || true
           k3d cluster delete test-domain-1 || true
           k3d registry delete k3d-registry.localhost || true
+
+  pr-tests-migrations:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: "clean .git/config"
+        if: matrix.os == 'windows-latest'
+        continue-on-error: true
+        shell: bash
+        run: |
+          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
+          rm ${GITHUB_WORKSPACE}/.git/config
+
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.syft == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pip packages
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv==0.2.17 tox tox-uv==1.9.0
+          uv --version
+
+      - name: Get uv cache dir
+        id: pip-cache
+        if: steps.changes.outputs.syft == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Load github cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.syft == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-py${{ matrix.python-version }}-
+
+      - name: Run migration tests
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          tox -e migration.test

.gitignore

@@ -6,7 +6,8 @@
 .idea/
 .mypy_cache
 .python-version
-.vscode/
+.vscode/*
+!.vscode/launch.json
 .tox/*
 .creds
 build
@@ -67,9 +68,15 @@ js/node_modules/*
 #nohup
 nohup.out
 
+# jupyter lsp
+.virtual_documents
+
 # notebook data
 notebooks/helm/scenario_data.jsonl
 
 # tox syft.build.helm generated file
 out.*
 .git-blame-ignore-revs
+
+# migration data
+packages/grid/helm/examples/dev/migration.yaml

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
         exclude: ^(packages/syft/tests/mongomock)
       - id: check-json
         always_run: true
-        exclude: ^(packages/grid/frontend/|packages/syft/tests/mongomock)
+        exclude: ^(packages/grid/frontend/|packages/syft/tests/mongomock|.vscode)
       - id: check-added-large-files
         always_run: true
         exclude: ^(packages/grid/backend/wheels/.*|docs/img/header.png|docs/img/terminalizer.gif)
@@ -179,7 +179,7 @@ repos:
     rev: "v3.0.0-alpha.9-for-vscode"
     hooks:
       - id: prettier
-        exclude: ^(packages/grid/helm|packages/grid/frontend/pnpm-lock.yaml|packages/syft/tests/mongomock)
+        exclude: ^(packages/grid/helm|packages/grid/frontend/pnpm-lock.yaml|packages/syft/tests/mongomock|.vscode)
 
   # - repo: meta
   #   hooks:

.vscode/launch.json

@@ -0,0 +1,25 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Syft Debugger",
+      "type": "debugpy",
+      "request": "attach",
+      "connect": {
+        "host": "localhost",
+        "port": "${input:port}"
+      }
+    }
+  ],
+  "inputs": [
+    {
+      "id": "port",
+      "description": "Port on which the debugger is listening",
+      "type": "promptString",
+      "default": "5678"
+    }
+  ]
+}

notebooks/admin/Custom API + Custom Worker.ipynb

@@ -91,7 +91,7 @@
     "custom_dockerfile_str = f\"\"\"\n",
     "FROM {registry}/openmined/grid-backend:{backend_version}\n",
     "\n",
-    "RUN pip install google-cloud-bigquery[all]==3.20.1 db-dtypes==1.2.0\n",
+    "RUN uv pip install google-cloud-bigquery[all]==3.20.1 db-dtypes==1.2.0\n",
     "\n",
     "\"\"\".strip()\n",
     "print(custom_dockerfile_str)"

notebooks/api/0.8/00-load-data.ipynb

@@ -348,7 +348,8 @@
    },
    "outputs": [],
    "source": [
-    "dataset.set_description(\"Canada Trade Data\")"
+    "dataset.set_description(\"Canada Trade Data Markdown Description\")\n",
+    "dataset.set_summary(\"Canada Trade Data Short Summary\")"
    ]
   },
   {

notebooks/api/0.8/02-review-code-and-approve.ipynb

@@ -325,7 +325,7 @@
    "outputs": [],
    "source": [
     "# Let's grab the actual executable function that was submitted by the user\n",
-    "users_function = func.unsafe_function"
+    "users_function = func.run"
    ]
   },
   {
@@ -501,7 +501,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.12.4"
   },
   "toc": {
    "base_numbering": 1,

notebooks/api/0.8/05-custom-policy.ipynb

@@ -507,7 +507,7 @@
    },
    "outputs": [],
    "source": [
-    "result = func.unsafe_function(x=x_pointer)\n",
+    "result = func.run(x=x_pointer)\n",
     "result"
    ]
   },
@@ -626,7 +626,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.12.4"
   },
   "toc": {
    "base_numbering": 1,

notebooks/api/0.8/10-container-images.ipynb

@@ -171,7 +171,7 @@
     "opendp_dockerfile_str = f\"\"\"\n",
     "FROM openmined/grid-backend:{syft_base_worker_tag}\n",
     "\n",
-    "RUN pip install opendp\n",
+    "RUN uv pip install opendp\n",
     "\n",
     "\"\"\".strip()\n",
     "\n",
@@ -761,7 +761,7 @@
    "outputs": [],
    "source": [
     "worker_delete_res = domain_client.api.services.worker.delete(\n",
-    "    uid=second_worker.id,\n",
+    "    uid=second_worker.id, force=True\n",
     ")"
    ]
   },
@@ -1081,7 +1081,7 @@
     "custom_dockerfile_str_2 = f\"\"\"\n",
     "FROM openmined/grid-backend:{syft_base_worker_tag}\n",
     "\n",
-    "RUN pip install opendp\n",
+    "RUN uv pip install opendp\n",
     "\"\"\".strip()\n",
     "\n",
     "docker_config_2 = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str_2)"
@@ -1297,7 +1297,7 @@
     "custom_dockerfile_str_3 = f\"\"\"\n",
     "FROM openmined/grid-backend:{syft_base_worker_tag}\n",
     "\n",
-    "RUN pip install recordlinkage\n",
+    "RUN uv pip install recordlinkage\n",
     "\"\"\".strip()\n",
     "\n",
     "docker_config_3 = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str_3)\n",
@@ -1319,7 +1319,7 @@
     "        num_workers=2,\n",
     "        tag=docker_tag_3,\n",
     "        config=docker_config_3,\n",
-    "        reason=\"I want to do some more cool data science with PySyft and OpenDP\",\n",
+    "        reason=\"I want to do some more cool data science with PySyft and recordlinkage\",\n",
     "        pull_image=pull,\n",
     "    )\n",
     ")\n",

notebooks/api/0.8/11-container-images-k8s.ipynb

@@ -265,7 +265,7 @@
     "custom_dockerfile_str = f\"\"\"\n",
     "FROM {registry}/{repo}:{tag}\n",
     "\n",
-    "RUN pip install pydicom\n",
+    "RUN uv pip install pydicom\n",
     "\n",
     "\"\"\".strip()"
    ]
@@ -811,6 +811,10 @@
     "    worker_pool_name=worker_pool_name,\n",
     ")\n",
     "def custom_worker_func(x):\n",
+    "    # third party\n",
+    "    import pydicom\n",
+    "\n",
+    "    print(pydicom.__version__)\n",
     "    return {\"y\": x + 1}"
    ]
   },
@@ -984,7 +988,7 @@
     "dockerfile_opendp = f\"\"\"\n",
     "FROM {registry}/{repo}:{tag}\n",
     "\n",
-    "RUN pip install opendp\n",
+    "RUN uv pip install opendp\n",
     "\"\"\".strip()\n",
     "\n",
     "docker_config_opendp = sy.DockerWorkerConfig(dockerfile=dockerfile_opendp)"
@@ -1279,7 +1283,7 @@
     "dockerfile_recordlinkage = f\"\"\"\n",
     "FROM {registry}/{repo}:{tag}\n",
     "\n",
-    "RUN pip install recordlinkage\n",
+    "RUN uv pip install recordlinkage\n",
     "\"\"\".strip()\n",
     "\n",
     "docker_config_recordlinkage = sy.DockerWorkerConfig(dockerfile=dockerfile_recordlinkage)\n",

notebooks/tutorials/data-owner/01-uploading-private-data.ipynb

@@ -124,8 +124,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "dataset_markdown_description = \"\"\"\n",
+    "### Contents\n",
+    "Numpy arrays of length 3 with integers ranging from 1 - 3.\n",
+    "\"\"\"\n",
     "dataset = sy.Dataset(\n",
     "    name=\"my dataset\",\n",
+    "    summary=\"Contains private and mock versions of data\",\n",
+    "    description=dataset_markdown_description,\n",
     "    asset_list=[\n",
     "        sy.Asset(name=\"my asset\", data=np.array([1, 2, 3]), mock=np.array([1, 1, 1]))\n",
     "    ],\n",

notebooks/tutorials/hello-syft/01-hello-syft.ipynb

@@ -412,7 +412,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "get_mean_age_user_function = usercode.unsafe_function"
+    "get_mean_age_user_function = usercode.run"
    ]
   },
   {

notebooks/tutorials/model-auditing/colab/01-user-log.ipynb

@@ -592,7 +592,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "real_result = request.code.unsafe_function(data=asset.data)\n",
+    "real_result = request.code.run(data=asset.data)\n",
     "real_result"
    ]
   },

notebooks/tutorials/model-training/02-data-owner-review-approve-code.ipynb

@@ -156,7 +156,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "users_function = user_code.unsafe_function\n",
+    "users_function = user_code.run\n",
     "users_function"
    ]
   },

notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb

@@ -738,7 +738,7 @@
    },
    "outputs": [],
    "source": [
-    "get_col_user_function = func.unsafe_function"
+    "get_col_user_function = func.run"
    ]
   },
   {