Merge branch 'dev' into fix/k8s-port

.github/workflows/cd-hagrid.yml

@@ -11,6 +11,12 @@ on:
         required: false
         default: "false"
 
+# Prevents concurrent runs of the same workflow
+# while the previous run is still in progress
+concurrency:
+  group: "CD - Hagrid"
+  cancel-in-progress: false
+
 jobs:
   call-pr-tests-linting:
     if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks

.github/workflows/cd-post-release-tests.yml

@@ -0,0 +1,55 @@
+name: CD - Post Release Tests
+
+on:
+  workflow_call:
+
+  workflow_dispatch:
+    inputs:
+      syft_version:
+        description: "Syft version to test"
+        required: true
+        type: string
+
+jobs:
+  syft-install-check:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.11", "3.10", "3.9"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install Syft
+        run: |
+          pip install syft==${{ github.event.inputs.syft_version }}
+
+      - name: Check Syft version
+        run: |
+          python -c "import syft; print(syft.__version__)"

.github/workflows/pr-tests-hagrid.yml

@@ -106,3 +106,65 @@ jobs:
         if: steps.changes.outputs.hagrid == 'true'
         run: |
           twine check dist/*.whl
+
+  pr-tests-syft-hagrid-comptability:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+        syft-version: ["0.8.2", "0.8.2b6", "0.8.3"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.hagrid == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        if: steps.changes.outputs.hagrid == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.hagrid == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('packages/syft/setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('packages/syft/setup.cfg') }}
+
+      # https://github.com/google/jax/issues/17693
+      # pinning ml-dtypes due to jax version==0.4.10
+      - name: Install Syft ${{ matrix.syft-version }}
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          pip install ml-dtypes==0.2.0
+          pip install syft==${{ matrix.syft-version }}
+          pip install .
+
+      - name: Run Orchestra Command
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          python -c "import syft as sy; domain1 = sy.orchestra.launch(name='test-domain-1', dev_mode=True, reset=True)"
+          python -c "import syft as sy; domain2 = sy.orchestra.launch(name='test-domain-2',dev_mode=False, reset=True)"

.pre-commit-config.yaml

@@ -172,7 +172,7 @@ repos:
       - id: mypy
         name: "mypy: syft"
         always_run: true
-        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py"
+        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py|^packages/syft/src/syft/service/warnings.py|^packages/syft/src/syft/service/dataset|^packages/syft/src/syft/service/worker"
         #files: "^packages/syft/src/syft/serde"
         args: [
             "--follow-imports=skip",

packages/hagrid/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.108
+current_version = 0.3.110
 tag = False
 tag_name = {new_version}
 commit = True

packages/hagrid/hagrid/manifest_template.yml

@@ -1,9 +1,9 @@
 manifestVersion: 0.1
-hagrid_version: 0.3.108
+hagrid_version: 0.3.110
 syft_version: 0.8.4-beta.22
 dockerTag: 0.8.4-beta.22
 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/
-hash: 799965e620b01f9cd1d8e7f23cc4cdc0fbc0410b
+hash: 7e36da6aac7dc6b63c8f23ad3d6d07cf51281362
 target_dir: ~/.hagrid/PySyft/
 files:
   grid:

packages/hagrid/hagrid/orchestra.py

@@ -13,6 +13,7 @@
 from typing import Any
 from typing import Callable
 from typing import Optional
+from typing import TYPE_CHECKING
 from typing import Union
 
 # relative
@@ -21,7 +22,6 @@
 from .names import random_name
 from .util import ImportFromSyft
 from .util import NodeSideType
-from .util import NodeType
 from .util import shell
 
 DEFAULT_PORT = 8080
@@ -31,6 +31,9 @@
 
 ClientAlias = Any  # we don't want to import Client in case it changes
 
+if TYPE_CHECKING:
+    NodeType = ImportFromSyft.import_node_type()
+
 
 # Define a function to read and print a stream
 def read_stream(stream: subprocess.PIPE) -> None:
@@ -95,6 +98,7 @@ def container_exists_with(name: str, port: int) -> bool:
 
 
 def get_node_type(node_type: Optional[Union[str, NodeType]]) -> Optional[NodeType]:
+    NodeType = ImportFromSyft.import_node_type()
     if node_type is None:
         node_type = os.environ.get("ORCHESTRA_NODE_TYPE", NodeType.DOMAIN)
     try:
@@ -236,6 +240,7 @@ def deploy_to_python(
     queue_port: Optional[int] = None,
 ) -> Optional[NodeHandle]:
     stage_protocol_changes = ImportFromSyft.import_stage_protocol_changes()
+    NodeType = ImportFromSyft.import_node_type()
     sy = get_syft_client()
     if sy is None:
         return sy
@@ -259,7 +264,7 @@ def deploy_to_python(
         "processes": processes,
         "dev_mode": dev_mode,
         "tail": tail,
-        "node_type": str(node_type_enum),
+        "node_type": node_type_enum,
         "node_side_type": node_side_type,
         "enable_warnings": enable_warnings,
         # new kwargs
@@ -296,7 +301,7 @@ def deploy_to_python(
             worker_class = worker_classes[node_type_enum]
             sig = inspect.signature(worker_class.named)
             supported_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters}
-            if "node_type" in sig.parameters.keys():
+            if "node_type" in sig.parameters.keys() and "migrate" in sig.parameters:
                 supported_kwargs["migrate"] = True
             worker = worker_class.named(**supported_kwargs)
         else:
@@ -482,6 +487,7 @@ def launch(
         queue_port: Optional[int] = None,
         in_memory_workers: bool = True,
     ) -> Optional[NodeHandle]:
+        NodeType = ImportFromSyft.import_node_type()
         if dev_mode is True:
             os.environ["DEV_MODE"] = "True"
             thread_workers = True
@@ -495,8 +501,6 @@ def launch(
         dev_mode = str_to_bool(os.environ.get("DEV_MODE", f"{dev_mode}"))
 
         node_type_enum: Optional[NodeType] = get_node_type(node_type=node_type)
-        if not node_type_enum:
-            return None
 
         node_side_type_enum = (
             NodeSideType.HIGH_SIDE

packages/hagrid/hagrid/util.py

@@ -22,17 +22,6 @@ def __str__(self) -> str:
         return self.value
 
 
-class NodeType(str, Enum):
-    DOMAIN = "domain"
-    NETWORK = "network"
-    ENCLAVE = "enclave"
-    GATEWAY = "gateway"
-
-    def __str__(self) -> str:
-        # Use values when transforming NodeType to str
-        return self.value
-
-
 class ImportFromSyft:
     @staticmethod
     def import_syft_error() -> Callable:
@@ -56,6 +45,16 @@ def stage_protocol_changes(*args: Any, **kwargs: Any) -> None:
 
         return stage_protocol_changes
 
+    @staticmethod
+    def import_node_type() -> Callable:
+        try:
+            # syft absolute
+            from syft.abstract_node import NodeType
+        except Exception:
+            NodeType = DummyNum
+
+        return NodeType
+
 
 def from_url(url: str) -> Tuple[str, str, int, str, Union[Any, str]]:
     try:

packages/hagrid/hagrid/version.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # HAGrid Version
-__version__ = "0.3.108"
+__version__ = "0.3.110"
 
 if __name__ == "__main__":
     print(__version__)

packages/hagrid/setup.py

@@ -5,7 +5,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = "0.3.108"
+__version__ = "0.3.110"
 
 DATA_FILES = {"img": ["hagrid/img/*.png"], "hagrid": ["*.yml"]}
 

packages/syft/src/syft/serde/third_party.py

@@ -96,8 +96,8 @@ def deserialize_dataframe(buf: bytes) -> DataFrame:
 
 
 def deserialize_series(blob: bytes) -> Series:
-    df = DataFrame.from_dict(deserialize(blob, from_bytes=True))
-    return df[df.columns[0]]
+    df: DataFrame = DataFrame.from_dict(deserialize(blob, from_bytes=True))
+    return Series(df[df.columns[0]])
 
 
 recursive_serde_register(