From fd25d5d84656c727d875689b801a514bf7677a61 Mon Sep 17 00:00:00 2001
From: khoaguin <dkn.work@protonmail.com>
Date: Tue, 6 Feb 2024 14:24:13 +0700
Subject: [PATCH 01/28] [refactor] fix mypy issues of syft/service/warnings.py

---
 .pre-commit-config.yaml                    |  2 +-
 packages/syft/src/syft/service/warnings.py | 99 +++++++++++-----------
 2 files changed, 50 insertions(+), 51 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 81b29307e8a..ffc8c73d05c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -172,7 +172,7 @@ repos:
       - id: mypy
         name: "mypy: syft"
         always_run: true
-        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py"
+        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py|^packages/syft/src/syft/service/warnings.py"
         #files: "^packages/syft/src/syft/serde"
         args: [
             "--follow-imports=skip",
diff --git a/packages/syft/src/syft/service/warnings.py b/packages/syft/src/syft/service/warnings.py
index a213cb63cb9..960496b0559 100644
--- a/packages/syft/src/syft/service/warnings.py
+++ b/packages/syft/src/syft/service/warnings.py
@@ -30,7 +30,7 @@ class APIEndpointWarning(SyftBaseModel):
     message: Optional[str] = None
     enabled: bool = True
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other: object) -> bool:
         if isinstance(other, APIEndpointWarning):
             return self.message == other.message and self._bool == other._bool
         return self._bool == other
@@ -54,7 +54,7 @@ def _repr_html_(self) -> str:
     def message_from(self, context: Optional[WarningContext]) -> Self:
         raise NotImplementedError
 
-    def show(self):
+    def show(self) -> bool:
         if not self.enabled or not self.message:
             return True
         display(self)
@@ -68,24 +68,24 @@ def show(self):
 
 @serializable()
 class CRUDWarning(APIEndpointWarning):
-    def message_from(self, context: Optional[WarningContext] = None):
+    def message_from(self, context: Optional[WarningContext] = None) -> Self:
         message = None
         confirmation = self.confirmation
         if context is not None:
             node = context.node
-            node_side_type = node.node_side_type
-            node_type = node.node_type
-            _msg = (
-                "which could host datasets with private information."
-                if node_side_type.value == NodeSideType.HIGH_SIDE.value
-                else "which only hosts mock or synthetic data."
-            )
-            message = (
-                "You're performing an operation on "
-                f"{node_side_type.value} side {node_type.value}, {_msg}"
-            )
-            confirmation = node_side_type.value == NodeSideType.HIGH_SIDE.value
-            message = message
+            if node is not None:
+                node_side_type = node.node_side_type
+                node_type = node.node_type
+                _msg = (
+                    "which could host datasets with private information."
+                    if node_side_type.value == NodeSideType.HIGH_SIDE.value
+                    else "which only hosts mock or synthetic data."
+                )
+                message = (
+                    "You're performing an operation on "
+                    f"{node_side_type.value} side {node_type.value}, {_msg}"
+                )
+                confirmation = node_side_type.value == NodeSideType.HIGH_SIDE.value
 
         return CRUDWarning(confirmation=confirmation, message=message)
 
@@ -94,63 +94,62 @@ def message_from(self, context: Optional[WarningContext] = None):
 class CRUDReminder(CRUDWarning):
     confirmation: bool = False
 
-    def message_from(self, context: Optional[WarningContext] = None):
+    def message_from(self, context: Optional[WarningContext] = None) -> Self:
         message = None
         confirmation = self.confirmation
         if context is not None:
             node = context.node
-            node_side_type = node.node_side_type
-            node_type = node.node_type
-            _msg = (
-                "which could host datasets with private information."
-                if node_side_type.value == NodeSideType.HIGH_SIDE.value
-                else "which only hosts mock or synthetic data."
-            )
-            message = (
-                "You're performing an operation on "
-                f"{node_side_type.value} side {node_type.value}, {_msg}"
-            )
-            message = message
+            if node is not None:
+                node_side_type = node.node_side_type
+                node_type = node.node_type
+                _msg = (
+                    "which could host datasets with private information."
+                    if node_side_type.value == NodeSideType.HIGH_SIDE.value
+                    else "which only hosts mock or synthetic data."
+                )
+                message = (
+                    "You're performing an operation on "
+                    f"{node_side_type.value} side {node_type.value}, {_msg}"
+                )
 
         return CRUDReminder(confirmation=confirmation, message=message)
 
 
 @serializable()
 class LowSideCRUDWarning(APIEndpointWarning):
-    def message_from(self, context: Optional[WarningContext] = None):
+    def message_from(self, context: Optional[WarningContext] = None) -> Self:
         confirmation = self.confirmation
         message = None
         if context is not None:
             node = context.node
-            node_side_type = node.node_side_type
-            node_type = node.node_type
-            if node_side_type.value == NodeSideType.LOW_SIDE.value:
-                message = (
-                    "You're performing an operation on "
-                    f"{node_side_type.value} side {node_type.value} "
-                    "which only hosts mock or synthetic data."
-                )
-
-            message = message
+            if node is not None:
+                node_side_type = node.node_side_type
+                node_type = node.node_type
+                if node_side_type.value == NodeSideType.LOW_SIDE.value:
+                    message = (
+                        "You're performing an operation on "
+                        f"{node_side_type.value} side {node_type.value} "
+                        "which only hosts mock or synthetic data."
+                    )
 
         return LowSideCRUDWarning(confirmation=confirmation, message=message)
 
 
 @serializable()
 class HighSideCRUDWarning(APIEndpointWarning):
-    def message_from(self, context: Optional[WarningContext] = None):
+    def message_from(self, context: Optional[WarningContext] = None) -> Self:
         confirmation = self.confirmation
         message = None
         if context is not None:
             node = context.node
-            node_side_type = node.node_side_type
-            node_type = node.node_type
-            if node_side_type.value == NodeSideType.HIGH_SIDE.value:
-                message = (
-                    "You're performing an operation on "
-                    f"{node_side_type.value} side {node_type.value} "
-                    "which could host datasets with private information."
-                )
-                message = message
+            if node is not None:
+                node_side_type = node.node_side_type
+                node_type = node.node_type
+                if node_side_type.value == NodeSideType.HIGH_SIDE.value:
+                    message = (
+                        "You're performing an operation on "
+                        f"{node_side_type.value} side {node_type.value} "
+                        "which could host datasets with private information."
+                    )
 
         return HighSideCRUDWarning(confirmation=confirmation, message=message)

From 0369157530e5b78047e0d0bbb6bb60fa57c8da18 Mon Sep 17 00:00:00 2001
From: khoaguin <dkn.work@protonmail.com>
Date: Wed, 7 Feb 2024 10:13:39 +0700
Subject: [PATCH 02/28] [refactor] fix mypy issues of `syft/service/dataset/`

---
 .pre-commit-config.yaml                       |  2 +-
 packages/syft/src/syft/serde/third_party.py   |  4 +-
 .../syft/src/syft/service/dataset/dataset.py  | 40 ++++++++++++-------
 .../syft/service/dataset/dataset_service.py   |  7 +++-
 4 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 81b29307e8a..116b1540163 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -172,7 +172,7 @@ repos:
       - id: mypy
         name: "mypy: syft"
         always_run: true
-        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py"
+        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py|^packages/syft/src/syft/service/dataset"
         #files: "^packages/syft/src/syft/serde"
         args: [
             "--follow-imports=skip",
diff --git a/packages/syft/src/syft/serde/third_party.py b/packages/syft/src/syft/serde/third_party.py
index 1abfe2d9cdc..2d70250cbe6 100644
--- a/packages/syft/src/syft/serde/third_party.py
+++ b/packages/syft/src/syft/serde/third_party.py
@@ -96,8 +96,8 @@ def deserialize_dataframe(buf: bytes) -> DataFrame:
 
 
 def deserialize_series(blob: bytes) -> Series:
-    df = DataFrame.from_dict(deserialize(blob, from_bytes=True))
-    return df[df.columns[0]]
+    df: DataFrame = DataFrame.from_dict(deserialize(blob, from_bytes=True))
+    return Series(df[df.columns[0]])
 
 
 recursive_serde_register(
diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py
index 87fed2a65b2..c62863fdc7d 100644
--- a/packages/syft/src/syft/service/dataset/dataset.py
+++ b/packages/syft/src/syft/service/dataset/dataset.py
@@ -101,7 +101,7 @@ class MarkdownDescription(SyftObject):
 
     text: str
 
-    def _repr_markdown_(self):
+    def _repr_markdown_(self) -> str:
         style = """
         <style>
             .jp-RenderedHTMLCommon pre {
@@ -138,7 +138,9 @@ class Asset(SyftObject):
     __repr_attrs__ = ["name", "shape"]
 
     def __init__(
-        self, description: Optional[Union[MarkdownDescription, str]] = "", **data
+        self,
+        description: Optional[Union[MarkdownDescription, str]] = "",
+        **data: Dict[str, Any],
     ):
         if isinstance(description, str):
             description = MarkdownDescription(text=description)
@@ -260,10 +262,10 @@ def mock(self) -> Union[SyftError, Any]:
         except Exception as e:
             return SyftError(message=f"Failed to get mock. {e}")
 
-    def has_data_permission(self):
+    def has_data_permission(self) -> bool:
         return self.data is not None
 
-    def has_permission(self, data_result):
+    def has_permission(self, data_result: Any) -> bool:
         # TODO: implement in a better way
         return not (
             isinstance(data_result, str)
@@ -329,7 +331,7 @@ class CreateAsset(SyftObject):
     class Config:
         validate_assignment = True
 
-    def __init__(self, description: Optional[str] = "", **data):
+    def __init__(self, description: Optional[str] = "", **data: Dict[str, Any]) -> None:
         super().__init__(**data, description=MarkdownDescription(text=str(description)))
 
     @root_validator()
@@ -470,14 +472,16 @@ class Dataset(SyftObject):
     __repr_attrs__ = ["name", "url", "created_at"]
 
     def __init__(
-        self, description: Optional[Union[str, MarkdownDescription]] = "", **data
-    ):
+        self,
+        description: Optional[Union[str, MarkdownDescription]] = "",
+        **data: Dict[str, Any],
+    ) -> None:
         if isinstance(description, str):
             description = MarkdownDescription(text=description)
         super().__init__(**data, description=description)
 
     @property
-    def icon(self):
+    def icon(self) -> str:
         return FOLDER_ICON
 
     def _coll_repr_(self) -> Dict[str, Any]:
@@ -498,7 +502,7 @@ def _repr_html_(self) -> Any:
             if self.uploader
             else ""
         )
-
+        description_text: str = self.description.text if self.description else ""
         return f"""
             <style>
             {fonts_css}
@@ -510,7 +514,7 @@ def _repr_html_(self) -> Any:
             </style>
             <div class='syft-dataset'>
             <h3>{self.name}</h3>
-            <p>{self.description.text}</p>
+            <p>{description_text}</p>
             {uploaded_by_line}
             <p class='paragraph-sm'><strong><span class='pr-8'>Created on: </span></strong>{self.created_at}</p>
             <p class='paragraph-sm'><strong><span class='pr-8'>URL:
@@ -535,7 +539,10 @@ def _old_repr_markdown_(self) -> str:
         _repr_str = f"Syft Dataset: {self.name}\n"
         _repr_str += "Assets:\n"
         for asset in self.asset_list:
-            _repr_str += f"\t{asset.name}: {asset.description.text}\n"
+            if asset.description is not None:
+                _repr_str += f"\t{asset.name}: {asset.description.text}\n\n"
+            else:
+                _repr_str += f"\t{asset.name}\n\n"
         if self.citation:
             _repr_str += f"Citation: {self.citation}\n"
         if self.url:
@@ -552,7 +559,10 @@ def _markdown_(self) -> str:
         _repr_str = f"Syft Dataset: {self.name}\n\n"
         _repr_str += "Assets:\n\n"
         for asset in self.asset_list:
-            _repr_str += f"\t{asset.name}: {asset.description.text}\n\n"
+            if asset.description is not None:
+                _repr_str += f"\t{asset.name}: {asset.description.text}\n\n"
+            else:
+                _repr_str += f"\t{asset.name}\n\n"
         if self.citation:
             _repr_str += f"Citation: {self.citation}\n\n"
         if self.url:
@@ -621,7 +631,7 @@ class CreateDataset(Dataset):
 
     id: Optional[UID] = None
     created_at: Optional[DateTime]
-    uploader: Optional[Contributor]
+    uploader: Optional[Contributor]  # type: ignore[assignment]
 
     class Config:
         validate_assignment = True
@@ -670,7 +680,7 @@ def add_contributor(
             return SyftError(message=f"Failed to add contributor. Error: {e}")
 
     def add_asset(
-        self, asset: CreateAsset, force_replace=False
+        self, asset: CreateAsset, force_replace: bool = False
     ) -> Union[SyftSuccess, SyftError]:
         if asset.mock is None:
             raise ValueError(_ASSET_WITH_NONE_MOCK_ERROR_MESSAGE)
@@ -694,7 +704,7 @@ def add_asset(
             message=f"Asset '{asset.name}' added to '{self.name}' Dataset."
         )
 
-    def replace_asset(self, asset: CreateAsset):
+    def replace_asset(self, asset: CreateAsset) -> Union[SyftSuccess, SyftError]:
         return self.add_asset(asset=asset, force_replace=True)
 
     def remove_asset(self, name: str) -> None:
diff --git a/packages/syft/src/syft/service/dataset/dataset_service.py b/packages/syft/src/syft/service/dataset/dataset_service.py
index 2558e1e560b..a06a3c72f74 100644
--- a/packages/syft/src/syft/service/dataset/dataset_service.py
+++ b/packages/syft/src/syft/service/dataset/dataset_service.py
@@ -2,6 +2,7 @@
 from collections.abc import Collection
 from typing import List
 from typing import Optional
+from typing import Sequence
 from typing import Union
 
 # relative
@@ -52,7 +53,7 @@ def _paginate_collection(
 
 
 def _paginate_dataset_collection(
-    datasets: Collection[Dataset],
+    datasets: Sequence[Dataset],
     page_size: Optional[int] = 0,
     page_index: Optional[int] = 0,
 ) -> Union[DictTuple[str, Dataset], DatasetPageView]:
@@ -204,7 +205,9 @@ def get_assets_by_action_id(
         name="dataset_delete_by_id",
         warning=HighSideCRUDWarning(confirmation=True),
     )
-    def delete_dataset(self, context: AuthedServiceContext, uid: UID):
+    def delete_dataset(
+        self, context: AuthedServiceContext, uid: UID
+    ) -> Union[SyftSuccess, SyftError]:
         result = self.stash.delete_by_uid(context.credentials, uid)
         if result.is_ok():
             return result.ok()

From 63d1653727488280325d8b63d9cdf2bd53e67e9e Mon Sep 17 00:00:00 2001
From: khoaguin <dkn.work@protonmail.com>
Date: Wed, 7 Feb 2024 11:05:23 +0700
Subject: [PATCH 03/28] [refactor] fixing mypy issues in `syft/service/worker`

---
 packages/syft/src/syft/serde/third_party.py   |   4 +-
 .../syft/src/syft/service/worker/utils.py     | 132 ++++++++++--------
 .../service/worker/worker_image_service.py    |  15 +-
 .../service/worker/worker_pool_service.py     |   6 +-
 .../src/syft/service/worker/worker_service.py |   7 +-
 5 files changed, 95 insertions(+), 69 deletions(-)

diff --git a/packages/syft/src/syft/serde/third_party.py b/packages/syft/src/syft/serde/third_party.py
index 1abfe2d9cdc..2d70250cbe6 100644
--- a/packages/syft/src/syft/serde/third_party.py
+++ b/packages/syft/src/syft/serde/third_party.py
@@ -96,8 +96,8 @@ def deserialize_dataframe(buf: bytes) -> DataFrame:
 
 
 def deserialize_series(blob: bytes) -> Series:
-    df = DataFrame.from_dict(deserialize(blob, from_bytes=True))
-    return df[df.columns[0]]
+    df: DataFrame = DataFrame.from_dict(deserialize(blob, from_bytes=True))
+    return Series(df[df.columns[0]])
 
 
 recursive_serde_register(
diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py
index 14b5799d825..a3dcc87d5bd 100644
--- a/packages/syft/src/syft/service/worker/utils.py
+++ b/packages/syft/src/syft/service/worker/utils.py
@@ -5,6 +5,8 @@
 import socket
 import socketserver
 import sys
+from typing import Any
+from typing import Dict
 from typing import List
 from typing import Optional
 from typing import Tuple
@@ -385,17 +387,22 @@ def run_workers_in_kubernetes(
     runner = KubernetesRunner()
 
     if start_idx == 0:
-        pool_pods = create_kubernetes_pool(
-            runner=runner,
-            tag=worker_image.image_identifier.full_name_with_tag,
-            pool_name=pool_name,
-            replicas=worker_count,
-            queue_port=queue_port,
-            debug=debug,
-            reg_username=reg_username,
-            reg_password=reg_password,
-            reg_url=reg_url,
-        )
+        if worker_image.image_identifier is not None:
+            pool_pods = create_kubernetes_pool(
+                runner=runner,
+                tag=worker_image.image_identifier.full_name_with_tag,
+                pool_name=pool_name,
+                replicas=worker_count,
+                queue_port=queue_port,
+                debug=debug,
+                reg_username=reg_username,
+                reg_password=reg_password,
+                reg_url=reg_url,
+            )
+        else:
+            return SyftError(
+                message=f"image with uid {worker_image.id} does not have an image identifier"
+            )
     else:
         pool_pods = scale_kubernetes_pool(runner, pool_name, worker_count)
 
@@ -584,28 +591,35 @@ def _get_healthcheck_based_on_status(status: WorkerStatus) -> WorkerHealth:
         return WorkerHealth.UNHEALTHY
 
 
-def image_build(image: SyftWorkerImage, **kwargs) -> Union[ImageBuildResult, SyftError]:
-    full_tag = image.image_identifier.full_name_with_tag
-    try:
-        builder = CustomWorkerBuilder()
-        return builder.build_image(
-            config=image.config,
-            tag=full_tag,
-            rm=True,
-            forcerm=True,
-            **kwargs,
-        )
-    except docker.errors.APIError as e:
-        return SyftError(
-            message=f"Docker API error when building '{full_tag}'. Reason - {e}"
-        )
-    except docker.errors.DockerException as e:
-        return SyftError(
-            message=f"Docker exception when building '{full_tag}'. Reason - {e}"
-        )
-    except Exception as e:
+def image_build(
+    image: SyftWorkerImage, **kwargs: Dict[str, Any]
+) -> Union[ImageBuildResult, SyftError]:
+    if image.image_identifier is not None:
+        full_tag = image.image_identifier.full_name_with_tag
+        try:
+            builder = CustomWorkerBuilder()
+            return builder.build_image(
+                config=image.config,
+                tag=full_tag,
+                rm=True,
+                forcerm=True,
+                **kwargs,
+            )
+        except docker.errors.APIError as e:
+            return SyftError(
+                message=f"Docker API error when building '{full_tag}'. Reason - {e}"
+            )
+        except docker.errors.DockerException as e:
+            return SyftError(
+                message=f"Docker exception when building '{full_tag}'. Reason - {e}"
+            )
+        except Exception as e:
+            return SyftError(
+                message=f"Unknown exception when building '{full_tag}'. Reason - {e}"
+            )
+    else:
         return SyftError(
-            message=f"Unknown exception when building '{full_tag}'. Reason - {e}"
+            message=f"image with uid {image.id} does not have an image identifier"
         )
 
 
@@ -614,34 +628,40 @@ def image_push(
     username: Optional[str] = None,
     password: Optional[str] = None,
 ) -> Union[ImagePushResult, SyftError]:
-    full_tag = image.image_identifier.full_name_with_tag
-    try:
-        builder = CustomWorkerBuilder()
-        result = builder.push_image(
-            # this should be consistent with docker build command
-            tag=image.image_identifier.full_name_with_tag,
-            registry_url=image.image_identifier.registry_host,
-            username=username,
-            password=password,
-        )
+    if image.image_identifier is not None:
+        full_tag = image.image_identifier.full_name_with_tag
+        try:
+            builder = CustomWorkerBuilder()
+            result = builder.push_image(
+                # this should be consistent with docker build command
+                tag=image.image_identifier.full_name_with_tag,
+                registry_url=image.image_identifier.registry_host,
+                username=username,
+                password=password,
+            )
 
-        if "error" in result.logs.lower() or result.exit_code:
+            if "error" in result.logs.lower() or result.exit_code:
+                return SyftError(
+                    message=f"Failed to push {full_tag}. "
+                    f"Exit code: {result.exit_code}. "
+                    f"Logs:\n{result.logs}"
+                )
+
+            return result
+        except docker.errors.APIError as e:
+            return SyftError(message=f"Docker API error when pushing {full_tag}. {e}")
+        except docker.errors.DockerException as e:
             return SyftError(
-                message=f"Failed to push {full_tag}. "
-                f"Exit code: {result.exit_code}. "
-                f"Logs:\n{result.logs}"
+                message=f"Docker exception when pushing {full_tag}. Reason - {e}"
             )
-
-        return result
-    except docker.errors.APIError as e:
-        return SyftError(message=f"Docker API error when pushing {full_tag}. {e}")
-    except docker.errors.DockerException as e:
-        return SyftError(
-            message=f"Docker exception when pushing {full_tag}. Reason - {e}"
-        )
-    except Exception as e:
+        except Exception as e:
+            return SyftError(
+                message=f"Unknown exception when pushing {image.image_identifier}. Reason - {e}"
+            )
+    else:
         return SyftError(
-            message=f"Unknown exception when pushing {image.image_identifier}. Reason - {e}"
+            message=f"image with uid {image.id} does not have an "
+            "image identifier and tag, hence we can't push it."
         )
 
 
diff --git a/packages/syft/src/syft/service/worker/worker_image_service.py b/packages/syft/src/syft/service/worker/worker_image_service.py
index 78b16a67395..8422341b009 100644
--- a/packages/syft/src/syft/service/worker/worker_image_service.py
+++ b/packages/syft/src/syft/service/worker/worker_image_service.py
@@ -75,7 +75,7 @@ def build(
         registry_uid: Optional[UID] = None,
         pull: bool = True,
     ) -> Union[SyftSuccess, SyftError]:
-        registry: SyftImageRegistry = None
+        registry: Optional[SyftImageRegistry] = None
 
         if IN_KUBERNETES and registry_uid is None:
             return SyftError(message="Registry UID is required in Kubernetes mode.")
@@ -96,7 +96,7 @@ def build(
             registry_result = image_registry_service.get_by_id(context, registry_uid)
             if registry_result.is_err():
                 return registry_result
-            registry: SyftImageRegistry = registry_result.ok()
+            registry = registry_result.ok()
 
         try:
             if registry:
@@ -204,12 +204,13 @@ def get_all(
         images: List[SyftWorkerImage] = result.ok()
 
         res = {}
-        # if image is built index by full_name_with_tag
-        res.update(
-            {im.image_identifier.full_name_with_tag: im for im in images if im.is_built}
-        )
+        # if image is built, index it by full_name_with_tag
+        for im in images:
+            if im.is_built and im.image_identifier is not None:
+                res[im.image_identifier.full_name_with_tag] = im
         # and then index all images by id
-        # TODO: jupyter repr needs to be updated to show unique values (even if multiple keys point to same value)
+        # TODO: jupyter repr needs to be updated to show unique values
+        # (even if multiple keys point to same value)
         res.update({im.id.to_string(): im for im in images if not im.is_built})
 
         return DictTuple(res)
diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py
index 806881547da..d2cd7db17c7 100644
--- a/packages/syft/src/syft/service/worker/worker_pool_service.py
+++ b/packages/syft/src/syft/service/worker/worker_pool_service.py
@@ -566,6 +566,10 @@ def _create_workers_in_pool(
             number=worker_cnt + existing_worker_cnt,
         )
     else:
+        if worker_image.image_identifier is not None:
+            registry_host = worker_image.image_identifier.registry_host
+        else:
+            registry_host = None
         result = run_containers(
             pool_name=pool_name,
             worker_image=worker_image,
@@ -576,7 +580,7 @@ def _create_workers_in_pool(
             dev_mode=context.node.dev_mode,
             reg_username=reg_username,
             reg_password=reg_password,
-            reg_url=worker_image.image_identifier.registry_host,
+            reg_url=registry_host,
         )
         if isinstance(result, SyftError):
             return result
diff --git a/packages/syft/src/syft/service/worker/worker_service.py b/packages/syft/src/syft/service/worker/worker_service.py
index 9871d7fa18a..af9b9ba7070 100644
--- a/packages/syft/src/syft/service/worker/worker_service.py
+++ b/packages/syft/src/syft/service/worker/worker_service.py
@@ -255,7 +255,7 @@ def refresh_worker_status(
     workers: List[SyftWorker],
     worker_stash: WorkerStash,
     credentials: SyftVerifyKey,
-):
+) -> List[SyftWorker]:
     if IN_KUBERNETES:
         result = refresh_status_kubernetes(workers)
     else:
@@ -277,7 +277,7 @@ def refresh_worker_status(
     return result
 
 
-def refresh_status_kubernetes(workers: List[SyftWorker]):
+def refresh_status_kubernetes(workers: List[SyftWorker]) -> List[SyftWorker]:
     updated_workers = []
     runner = KubernetesRunner()
     for worker in workers:
@@ -292,7 +292,7 @@ def refresh_status_kubernetes(workers: List[SyftWorker]):
     return updated_workers
 
 
-def refresh_status_docker(workers: List[SyftWorker]):
+def refresh_status_docker(workers: List[SyftWorker]) -> List[SyftWorker]:
     updated_workers = []
 
     with contextlib.closing(docker.from_env()) as client:
@@ -317,6 +317,7 @@ def _stop_worker_container(
         container.stop()
         # Remove the container and its volumes
         _remove_worker_container(container, force=force, v=True)
+        return None
     except Exception as e:
         return SyftError(
             message=f"Failed to delete worker with id: {worker.id}. Error: {e}"

From db1ff85149b8eb55a2ff9f6122e8c1ac098edf7d Mon Sep 17 00:00:00 2001
From: khoaguin <dkn.work@protonmail.com>
Date: Wed, 7 Feb 2024 15:16:34 +0700
Subject: [PATCH 04/28] [refactor] done fixing mypy issues in
 `syft/service/worker`

---
 .pre-commit-config.yaml                       |  2 +-
 .../syft/service/worker/image_identifier.py   |  2 +-
 .../src/syft/service/worker/image_registry.py |  4 +-
 .../service/worker/image_registry_service.py  |  2 +-
 .../syft/src/syft/service/worker/utils.py     | 53 ++++++++++++-------
 .../syft/src/syft/service/worker/worker.py    | 10 ++--
 .../syft/service/worker/worker_image_stash.py |  3 +-
 .../src/syft/service/worker/worker_pool.py    | 12 ++++-
 .../src/syft/service/worker/worker_stash.py   |  4 +-
 9 files changed, 60 insertions(+), 32 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 81b29307e8a..0e1aecdd0ce 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -172,7 +172,7 @@ repos:
       - id: mypy
         name: "mypy: syft"
         always_run: true
-        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py"
+        files: "^packages/syft/src/syft/serde|^packages/syft/src/syft/util/env.py|^packages/syft/src/syft/util/logger.py|^packages/syft/src/syft/util/markdown.py|^packages/syft/src/syft/util/notebook_ui/notebook_addons.py|^packages/syft/src/syft/service/worker"
         #files: "^packages/syft/src/syft/serde"
         args: [
             "--follow-imports=skip",
diff --git a/packages/syft/src/syft/service/worker/image_identifier.py b/packages/syft/src/syft/service/worker/image_identifier.py
index 43623f44d36..4651c3f4f2e 100644
--- a/packages/syft/src/syft/service/worker/image_identifier.py
+++ b/packages/syft/src/syft/service/worker/image_identifier.py
@@ -53,7 +53,7 @@ def from_str(cls, tag: str) -> Self:
         return cls(repo=repo, registry=registry, tag=tag)
 
     @property
-    def repo_with_tag(self) -> str:
+    def repo_with_tag(self) -> Optional[str]:
         if self.repo or self.tag:
             return f"{self.repo}:{self.tag}"
         return None
diff --git a/packages/syft/src/syft/service/worker/image_registry.py b/packages/syft/src/syft/service/worker/image_registry.py
index 806a0946d2b..7fb9dfcb770 100644
--- a/packages/syft/src/syft/service/worker/image_registry.py
+++ b/packages/syft/src/syft/service/worker/image_registry.py
@@ -28,7 +28,7 @@ class SyftImageRegistry(SyftObject):
     url: str
 
     @validator("url")
-    def validate_url(cls, val: str):
+    def validate_url(cls, val: str) -> str:
         if not val:
             raise ValueError("Invalid Registry URL. Must not be empty")
 
@@ -38,7 +38,7 @@ def validate_url(cls, val: str):
         return val
 
     @classmethod
-    def from_url(cls, full_str: str):
+    def from_url(cls, full_str: str) -> "SyftImageRegistry":
         # this is only for urlparse
         if "://" not in full_str:
             full_str = f"http://{full_str}"
diff --git a/packages/syft/src/syft/service/worker/image_registry_service.py b/packages/syft/src/syft/service/worker/image_registry_service.py
index 8c628f84486..81aa276559c 100644
--- a/packages/syft/src/syft/service/worker/image_registry_service.py
+++ b/packages/syft/src/syft/service/worker/image_registry_service.py
@@ -62,7 +62,7 @@ def delete(
         self,
         context: AuthedServiceContext,
         uid: UID = None,
-        url: str = None,
+        url: str = "",
     ) -> Union[SyftSuccess, SyftError]:
         # TODO - we need to make sure that there are no workers running an image bound to this registry
 
diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py
index a3dcc87d5bd..fafe95f3112 100644
--- a/packages/syft/src/syft/service/worker/utils.py
+++ b/packages/syft/src/syft/service/worker/utils.py
@@ -14,6 +14,8 @@
 
 # third party
 import docker
+from docker.models.containers import Container
+from kr8s.objects import Pod
 
 # relative
 from ...abstract_node import AbstractNode
@@ -49,7 +51,9 @@ def backend_container_name() -> str:
     return f"{hostname}-{service_name}-1"
 
 
-def get_container(docker_client: docker.DockerClient, container_name: str):
+def get_container(
+    docker_client: docker.DockerClient, container_name: str
+) -> Optional[Container]:
     try:
         existing_container = docker_client.containers.get(container_name)
     except docker.errors.NotFound:
@@ -58,14 +62,20 @@ def get_container(docker_client: docker.DockerClient, container_name: str):
     return existing_container
 
 
-def extract_config_from_backend(worker_name: str, docker_client: docker.DockerClient):
+def extract_config_from_backend(
+    worker_name: str, docker_client: docker.DockerClient
+) -> Dict[str, Any]:
     # Existing main backend container
     backend_container = get_container(
         docker_client, container_name=backend_container_name()
     )
 
     # Config with defaults
-    extracted_config = {"volume_binds": {}, "network_mode": None, "environment": {}}
+    extracted_config: Dict[str, Any] = {
+        "volume_binds": {},
+        "network_mode": None,
+        "environment": {},
+    }
 
     if backend_container is None:
         return extracted_config
@@ -96,7 +106,7 @@ def extract_config_from_backend(worker_name: str, docker_client: docker.DockerCl
     return extracted_config
 
 
-def get_free_tcp_port():
+def get_free_tcp_port() -> int:
     with socketserver.TCPServer(("localhost", 0), None) as s:
         free_port = s.server_address[1]
         return free_port
@@ -115,7 +125,7 @@ def run_container_using_docker(
     registry_url: Optional[str] = None,
 ) -> ContainerSpawnStatus:
     if not worker_image.is_built:
-        raise Exception("Image must be built before running it.")
+        raise ValueError("Image must be built before running it.")
 
     # Get hostname
     hostname = socket.gethostname()
@@ -167,8 +177,11 @@ def run_container_using_docker(
         environment["QUEUE_PORT"] = queue_port
         environment["CONTAINER_HOST"] = "docker"
 
+        if worker_image.image_identifier is None:
+            raise ValueError(f"Image {worker_image} does not have an identifier")
+
         container = docker_client.containers.run(
-            worker_image.image_identifier.full_name_with_tag,
+            image=worker_image.image_identifier.full_name_with_tag,
             name=f"{hostname}-{worker_name}",
             detach=True,
             auto_remove=True,
@@ -259,20 +272,22 @@ def run_workers_in_threads(
     return results
 
 
-def prepare_kubernetes_pool_env(runner: KubernetesRunner, env_vars: dict):
+def prepare_kubernetes_pool_env(
+    runner: KubernetesRunner, env_vars: dict
+) -> Tuple[List, Dict]:
     # get current backend pod name
     backend_pod_name = os.getenv("K8S_POD_NAME")
     if not backend_pod_name:
-        raise ValueError(message="Pod name not provided in environment variable")
+        raise ValueError("Pod name not provided in environment variable")
 
     # get current backend's credentials path
-    creds_path = os.getenv("CREDENTIALS_PATH")
+    creds_path: Union[str, None, Path] = os.getenv("CREDENTIALS_PATH")
     if not creds_path:
-        raise ValueError(message="Credentials path not provided")
+        raise ValueError("Credentials path not provided")
 
     creds_path = Path(creds_path)
-    if not creds_path.exists():
-        raise ValueError(message="Credentials file does not exist")
+    if creds_path is not None and not creds_path.exists():
+        raise ValueError("Credentials file does not exist")
 
     # create a secret for the node credentials owned by the backend, not the pool.
     node_secret = KubeUtils.create_secret(
@@ -285,7 +300,7 @@ def prepare_kubernetes_pool_env(runner: KubernetesRunner, env_vars: dict):
 
     # clone and patch backend environment variables
     backend_env = runner.get_pod_env_vars(backend_pod_name) or []
-    env_vars = KubeUtils.patch_env_vars(backend_env, env_vars)
+    env_vars_list = KubeUtils.patch_env_vars(backend_env, env_vars)
     mount_secrets = {
         node_secret.metadata.name: {
             "mountPath": str(creds_path),
@@ -293,7 +308,7 @@ def prepare_kubernetes_pool_env(runner: KubernetesRunner, env_vars: dict):
         },
     }
 
-    return env_vars, mount_secrets
+    return env_vars_list, mount_secrets
 
 
 def create_kubernetes_pool(
@@ -306,8 +321,8 @@ def create_kubernetes_pool(
     reg_username: Optional[str] = None,
     reg_password: Optional[str] = None,
     reg_url: Optional[str] = None,
-    **kwargs,
-):
+    **kwargs: Dict[str, Any],
+) -> Union[SyftError, List[Pod]]:
     pool = None
     error = False
 
@@ -357,7 +372,7 @@ def scale_kubernetes_pool(
     runner: KubernetesRunner,
     pool_name: str,
     replicas: int,
-):
+) -> Union[SyftError, List[Pod]]:
     pool = runner.get_pool(pool_name)
     if not pool:
         return SyftError(message=f"Pool does not exist. name={pool_name}")
@@ -376,12 +391,12 @@ def run_workers_in_kubernetes(
     worker_count: int,
     pool_name: str,
     queue_port: int,
-    start_idx=0,
+    start_idx: int = 0,
     debug: bool = False,
     reg_username: Optional[str] = None,
     reg_password: Optional[str] = None,
     reg_url: Optional[str] = None,
-    **kwargs,
+    **kwargs: Dict[str, Any],
 ) -> Union[List[ContainerSpawnStatus], SyftError]:
     spawn_status = []
     runner = KubernetesRunner()
diff --git a/packages/syft/src/syft/service/worker/worker.py b/packages/syft/src/syft/service/worker/worker.py
index 0242dba177a..ef3fc4aec5d 100644
--- a/packages/syft/src/syft/service/worker/worker.py
+++ b/packages/syft/src/syft/service/worker/worker.py
@@ -1,4 +1,8 @@
 # stdlib
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
 
 # relative
 from ...serde.serializable import serializable
@@ -39,7 +43,7 @@ class DockerWorker(SyftObject):
     container_id: str
     created_at: DateTime = DateTime.now()
 
-    def _coll_repr_(self):
+    def _coll_repr_(self) -> Dict[str, Any]:
         return {
             "container_name": self.container_name,
             "container_id": self.container_id,
@@ -48,10 +52,10 @@ def _coll_repr_(self):
 
 
 @migrate(DockerWorker, DockerWorkerV1)
-def downgrade_job_v2_to_v1():
+def downgrade_job_v2_to_v1() -> List[Callable]:
     return [drop(["container_name"])]
 
 
 @migrate(DockerWorkerV1, DockerWorker)
-def upgrade_job_v2_to_v3():
+def upgrade_job_v2_to_v3() -> List[Callable]:
     return [make_set_default("job_consumer_id", None)]
diff --git a/packages/syft/src/syft/service/worker/worker_image_stash.py b/packages/syft/src/syft/service/worker/worker_image_stash.py
index 49eb6fa1802..a1580076104 100644
--- a/packages/syft/src/syft/service/worker/worker_image_stash.py
+++ b/packages/syft/src/syft/service/worker/worker_image_stash.py
@@ -1,5 +1,6 @@
 # stdlib
 from typing import List
+from typing import Optional
 from typing import Union
 
 # third party
@@ -59,6 +60,6 @@ def set(
 
     def get_by_docker_config(
         self, credentials: SyftVerifyKey, config: DockerWorkerConfig
-    ):
+    ) -> Result[Optional[SyftWorkerImage], str]:
         qks = QueryKeys(qks=[WorkerConfigPK.with_obj(config)])
         return self.query_one(credentials=credentials, qks=qks)
diff --git a/packages/syft/src/syft/service/worker/worker_pool.py b/packages/syft/src/syft/service/worker/worker_pool.py
index 377ee118b55..14f9ff0a7dd 100644
--- a/packages/syft/src/syft/service/worker/worker_pool.py
+++ b/packages/syft/src/syft/service/worker/worker_pool.py
@@ -87,7 +87,7 @@ def logs(self) -> Union[str, SyftError]:
 
         return api.services.worker.logs(uid=self.id)
 
-    def get_job_repr(self):
+    def get_job_repr(self) -> str:
         if self.job_id is not None:
             api = APIRegistry.api_for(
                 node_uid=self.syft_node_location,
@@ -117,14 +117,20 @@ def refresh_status(self) -> None:
 
     def _coll_repr_(self) -> Dict[str, Any]:
         self.refresh_status()
+
         if self.image and self.image.image_identifier:
             image_name_with_tag = self.image.image_identifier.full_name_with_tag
         else:
             image_name_with_tag = "In Memory Worker"
+
+        healthcheck: str = ""
+        if self.healthcheck is not None:
+            healthcheck = self.healthcheck.value
+
         return {
             "Name": self.name,
             "Image": image_name_with_tag,
-            "Healthcheck (health / unhealthy)": f"{self.healthcheck.value}",
+            "Healthcheck (health / unhealthy)": f"{healthcheck}",
             "Status": f"{self.status.value}",
             "Job": self.get_job_repr(),
             "Created at": str(self.created_at),
@@ -166,6 +172,8 @@ def image(self) -> Optional[Union[SyftWorkerImage, SyftError]]:
         )
         if api is not None:
             return api.services.worker_image.get_by_uid(uid=self.image_id)
+        else:
+            return None
 
     @property
     def running_workers(self) -> Union[List[UID], SyftError]:
diff --git a/packages/syft/src/syft/service/worker/worker_stash.py b/packages/syft/src/syft/service/worker/worker_stash.py
index 1e82dcfccdf..4a3abbbeaa1 100644
--- a/packages/syft/src/syft/service/worker/worker_stash.py
+++ b/packages/syft/src/syft/service/worker/worker_stash.py
@@ -59,13 +59,13 @@ def get_worker_by_name(
 
     def update_consumer_state(
         self, credentials: SyftVerifyKey, worker_uid: UID, consumer_state: ConsumerState
-    ):
+    ) -> Result[Ok, Err]:
         res = self.get_by_uid(credentials=credentials, uid=worker_uid)
         if res.is_err():
             return Err(
                 f"Failed to retrieve Worker with id: {worker_uid}. Error: {res.err()}"
             )
-        worker: SyftWorker = res.ok()
+        worker: Optional[SyftWorker] = res.ok()
         if worker is None:
             return Err(f"Worker with id: {worker_uid} not found")
         worker.consumer_state = consumer_state

From bcf2aa5117d59aa097910b041de452b4b38bfb1c Mon Sep 17 00:00:00 2001
From: khoaguin <dkn.work@protonmail.com>
Date: Mon, 12 Feb 2024 09:57:24 +0700
Subject: [PATCH 05/28] [refactor] fix some issues according to comments

Co-authored-by: Kien Dang <mail@kien.ai>
---
 .../src/syft/service/worker/image_registry_service.py  |  3 ++-
 packages/syft/src/syft/service/worker/utils.py         | 10 +++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/packages/syft/src/syft/service/worker/image_registry_service.py b/packages/syft/src/syft/service/worker/image_registry_service.py
index 81aa276559c..8acbd9e6f4b 100644
--- a/packages/syft/src/syft/service/worker/image_registry_service.py
+++ b/packages/syft/src/syft/service/worker/image_registry_service.py
@@ -1,5 +1,6 @@
 # stdlib
 from typing import List
+from typing import Optional
 from typing import Union
 
 # relative
@@ -62,7 +63,7 @@ def delete(
         self,
         context: AuthedServiceContext,
         uid: UID = None,
-        url: str = "",
+        url: Optional[str] = None,
     ) -> Union[SyftSuccess, SyftError]:
         # TODO - we need to make sure that there are no workers running an image bound to this registry
 
diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py
index fafe95f3112..4c50e3dd764 100644
--- a/packages/syft/src/syft/service/worker/utils.py
+++ b/packages/syft/src/syft/service/worker/utils.py
@@ -281,7 +281,7 @@ def prepare_kubernetes_pool_env(
         raise ValueError("Pod name not provided in environment variable")
 
     # get current backend's credentials path
-    creds_path: Union[str, None, Path] = os.getenv("CREDENTIALS_PATH")
+    creds_path: Optional[Union[str, Path]] = os.getenv("CREDENTIALS_PATH")
     if not creds_path:
         raise ValueError("Credentials path not provided")
 
@@ -300,7 +300,7 @@ def prepare_kubernetes_pool_env(
 
     # clone and patch backend environment variables
     backend_env = runner.get_pod_env_vars(backend_pod_name) or []
-    env_vars_list = KubeUtils.patch_env_vars(backend_env, env_vars)
+    env_vars_: List = KubeUtils.patch_env_vars(backend_env, env_vars)
     mount_secrets = {
         node_secret.metadata.name: {
             "mountPath": str(creds_path),
@@ -308,7 +308,7 @@ def prepare_kubernetes_pool_env(
         },
     }
 
-    return env_vars_list, mount_secrets
+    return env_vars_, mount_secrets
 
 
 def create_kubernetes_pool(
@@ -321,7 +321,7 @@ def create_kubernetes_pool(
     reg_username: Optional[str] = None,
     reg_password: Optional[str] = None,
     reg_url: Optional[str] = None,
-    **kwargs: Dict[str, Any],
+    **kwargs: Any,
 ) -> Union[SyftError, List[Pod]]:
     pool = None
     error = False
@@ -396,7 +396,7 @@ def run_workers_in_kubernetes(
     reg_username: Optional[str] = None,
     reg_password: Optional[str] = None,
     reg_url: Optional[str] = None,
-    **kwargs: Dict[str, Any],
+    **kwargs: Any,
 ) -> Union[List[ContainerSpawnStatus], SyftError]:
     spawn_status = []
     runner = KubernetesRunner()

From f523fc9506b505286c4730f326c15ca051308b0d Mon Sep 17 00:00:00 2001
From: khoaguin <dkn.work@protonmail.com>
Date: Mon, 12 Feb 2024 10:10:45 +0700
Subject: [PATCH 06/28] [refactor] change `**data: Dict[str, Any]` to `**data:
 Any` according to comments

Co-authored-by: Kien Dang <mail@kien.ai>
---
 packages/syft/src/syft/service/dataset/dataset.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py
index c62863fdc7d..a5d56e8a605 100644
--- a/packages/syft/src/syft/service/dataset/dataset.py
+++ b/packages/syft/src/syft/service/dataset/dataset.py
@@ -140,7 +140,7 @@ class Asset(SyftObject):
     def __init__(
         self,
         description: Optional[Union[MarkdownDescription, str]] = "",
-        **data: Dict[str, Any],
+        **data: Any,
     ):
         if isinstance(description, str):
             description = MarkdownDescription(text=description)
@@ -331,7 +331,7 @@ class CreateAsset(SyftObject):
     class Config:
         validate_assignment = True
 
-    def __init__(self, description: Optional[str] = "", **data: Dict[str, Any]) -> None:
+    def __init__(self, description: Optional[str] = "", **data: Any) -> None:
         super().__init__(**data, description=MarkdownDescription(text=str(description)))
 
     @root_validator()
@@ -474,7 +474,7 @@ class Dataset(SyftObject):
     def __init__(
         self,
         description: Optional[Union[str, MarkdownDescription]] = "",
-        **data: Dict[str, Any],
+        **data: Any,
     ) -> None:
         if isinstance(description, str):
             description = MarkdownDescription(text=description)

From 22e8f73942e6fb3fc9f43d504dbe90d7dabda27c Mon Sep 17 00:00:00 2001
From: khoaguin <dkn.work@protonmail.com>
Date: Tue, 13 Feb 2024 10:05:30 +0700
Subject: [PATCH 07/28] [refactor] change arg type from `object` to `Any`

---
 packages/syft/src/syft/service/warnings.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/syft/src/syft/service/warnings.py b/packages/syft/src/syft/service/warnings.py
index 960496b0559..553531046a8 100644
--- a/packages/syft/src/syft/service/warnings.py
+++ b/packages/syft/src/syft/service/warnings.py
@@ -1,4 +1,5 @@
 # stdlib
+from typing import Any
 from typing import Optional
 
 # third party
@@ -30,7 +31,7 @@ class APIEndpointWarning(SyftBaseModel):
     message: Optional[str] = None
     enabled: bool = True
 
-    def __eq__(self, other: object) -> bool:
+    def __eq__(self, other: Any) -> bool:
         if isinstance(other, APIEndpointWarning):
             return self.message == other.message and self._bool == other._bool
         return self._bool == other

From cd817a755c85fea22b1db7111141ec0060606122 Mon Sep 17 00:00:00 2001
From: khoaguin <dkn.work@protonmail.com>
Date: Wed, 14 Feb 2024 14:25:56 +0700
Subject: [PATCH 08/28] [refactor] refactor some functions and their return
 types

---
 .../syft/src/syft/service/worker/worker_pool_service.py   | 8 +++++---
 packages/syft/src/syft/service/worker/worker_stash.py     | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py
index db3465e57e5..3680ab20346 100644
--- a/packages/syft/src/syft/service/worker/worker_pool_service.py
+++ b/packages/syft/src/syft/service/worker/worker_pool_service.py
@@ -443,7 +443,7 @@ def scale(
         number: int,
         pool_id: Optional[UID] = None,
         pool_name: Optional[str] = None,
-    ) -> Union[SyftError, SyftSuccess, List[ContainerSpawnStatus]]:
+    ) -> Union[SyftError, SyftSuccess]:
         """
         Scale the worker pool to the given number of workers in Kubernetes.
         Allows both scaling up and down the worker pool.
@@ -466,7 +466,7 @@ def scale(
             return SyftSuccess(message=f"Worker pool already has {number} workers")
         elif number > current_worker_count:
             workers_to_add = number - current_worker_count
-            return self.add_workers(
+            result = self.add_workers(
                 context=context,
                 number=workers_to_add,
                 pool_id=pool_id,
@@ -475,6 +475,8 @@ def scale(
                 reg_username=None,
                 reg_password=None,
             )
+            if isinstance(result, SyftError):
+                return result
         else:
             # scale down at kubernetes control plane
             runner = KubernetesRunner()
@@ -514,7 +516,7 @@ def scale(
                 return SyftError(
                     message=(
                         f"Pool {worker_pool.name} was scaled down, "
-                        f"but failed update the stash with err: {result.err()}"
+                        f"but failed update the stash with err: {update_result.err()}"
                     )
                 )
 
diff --git a/packages/syft/src/syft/service/worker/worker_stash.py b/packages/syft/src/syft/service/worker/worker_stash.py
index 4a3abbbeaa1..cb7a914ed9b 100644
--- a/packages/syft/src/syft/service/worker/worker_stash.py
+++ b/packages/syft/src/syft/service/worker/worker_stash.py
@@ -59,7 +59,7 @@ def get_worker_by_name(
 
     def update_consumer_state(
         self, credentials: SyftVerifyKey, worker_uid: UID, consumer_state: ConsumerState
-    ) -> Result[Ok, Err]:
+    ) -> Result[str, str]:
         res = self.get_by_uid(credentials=credentials, uid=worker_uid)
         if res.is_err():
             return Err(

From c6185b16d2089459cb21b68fa29e68f77d21a100 Mon Sep 17 00:00:00 2001
From: Kien Dang <mail@kien.ai>
Date: Wed, 14 Feb 2024 15:50:01 +0800
Subject: [PATCH 09/28] Switch success/error type order

---
 packages/syft/src/syft/service/worker/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py
index 8681cebd7f6..f7443139027 100644
--- a/packages/syft/src/syft/service/worker/utils.py
+++ b/packages/syft/src/syft/service/worker/utils.py
@@ -322,7 +322,7 @@ def create_kubernetes_pool(
     reg_password: Optional[str] = None,
     reg_url: Optional[str] = None,
     **kwargs: Any,
-) -> Union[SyftError, List[Pod]]:
+) -> Union[List[Pod], SyftError]:
     pool = None
     error = False
 
@@ -372,7 +372,7 @@ def scale_kubernetes_pool(
     runner: KubernetesRunner,
     pool_name: str,
     replicas: int,
-) -> Union[SyftError, List[Pod]]:
+) -> Union[List[Pod], SyftError]:
     pool = runner.get_pool(pool_name)
     if not pool:
         return SyftError(message=f"Pool does not exist. name={pool_name}")

From 558edb423519d4299e63bc90e456e66e4a3ae3ef Mon Sep 17 00:00:00 2001
From: Kien Dang <mail@kien.ai>
Date: Wed, 14 Feb 2024 15:51:41 +0800
Subject: [PATCH 10/28] Use typing.Self instead of hard-coded class

---
 packages/syft/src/syft/service/worker/image_registry.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/syft/src/syft/service/worker/image_registry.py b/packages/syft/src/syft/service/worker/image_registry.py
index 7fb9dfcb770..7292273c605 100644
--- a/packages/syft/src/syft/service/worker/image_registry.py
+++ b/packages/syft/src/syft/service/worker/image_registry.py
@@ -4,6 +4,7 @@
 
 # third party
 from pydantic import validator
+from typing_extensions import Self
 
 # relative
 from ...serde.serializable import serializable
@@ -38,7 +39,7 @@ def validate_url(cls, val: str) -> str:
         return val
 
     @classmethod
-    def from_url(cls, full_str: str) -> "SyftImageRegistry":
+    def from_url(cls, full_str: str) -> Self:
         # this is only for urlparse
         if "://" not in full_str:
             full_str = f"http://{full_str}"

From df7cfef3bc09c8c98640990b18a70270ad87f821 Mon Sep 17 00:00:00 2001
From: Kien Dang <mail@kien.ai>
Date: Wed, 14 Feb 2024 17:36:35 +0800
Subject: [PATCH 11/28] Use ternary for simplicity

---
 packages/syft/src/syft/service/worker/worker_pool.py     | 4 +---
 .../syft/src/syft/service/worker/worker_pool_service.py  | 9 +++++----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/packages/syft/src/syft/service/worker/worker_pool.py b/packages/syft/src/syft/service/worker/worker_pool.py
index 14f9ff0a7dd..151addcaac0 100644
--- a/packages/syft/src/syft/service/worker/worker_pool.py
+++ b/packages/syft/src/syft/service/worker/worker_pool.py
@@ -123,9 +123,7 @@ def _coll_repr_(self) -> Dict[str, Any]:
         else:
             image_name_with_tag = "In Memory Worker"
 
-        healthcheck: str = ""
-        if self.healthcheck is not None:
-            healthcheck = self.healthcheck.value
+        healthcheck = self.healthcheck.value if self.healthcheck is not None else ""
 
         return {
             "Name": self.name,
diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py
index 3680ab20346..38f223332ea 100644
--- a/packages/syft/src/syft/service/worker/worker_pool_service.py
+++ b/packages/syft/src/syft/service/worker/worker_pool_service.py
@@ -658,10 +658,11 @@ def _create_workers_in_pool(
             number=worker_cnt + existing_worker_cnt,
         )
     else:
-        if worker_image.image_identifier is not None:
-            registry_host = worker_image.image_identifier.registry_host
-        else:
-            registry_host = None
+        registry_host = (
+            worker_image.image_identifier.registry_host
+            if worker_image.image_identifier is not None
+            else None
+        )
         result = run_containers(
             pool_name=pool_name,
             worker_image=worker_image,

From 1831635cc039499fb20ee89936aa9e4f7e978522 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 15:57:43 +0530
Subject: [PATCH 12/28] fixed node type string errors in previous syft versions

---
 packages/hagrid/hagrid/orchestra.py | 14 ++++++++++----
 packages/hagrid/hagrid/util.py      | 21 ++++++++++-----------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py
index 8576b8fef7c..4ff0d26293b 100644
--- a/packages/hagrid/hagrid/orchestra.py
+++ b/packages/hagrid/hagrid/orchestra.py
@@ -13,6 +13,7 @@
 from typing import Any
 from typing import Callable
 from typing import Optional
+from typing import TYPE_CHECKING
 from typing import Union
 
 # relative
@@ -21,7 +22,6 @@
 from .names import random_name
 from .util import ImportFromSyft
 from .util import NodeSideType
-from .util import NodeType
 from .util import shell
 
 DEFAULT_PORT = 8080
@@ -31,6 +31,9 @@
 
 ClientAlias = Any  # we don't want to import Client in case it changes
 
+if TYPE_CHECKING:
+    NodeType = ImportFromSyft.import_node_type()
+
 
 # Define a function to read and print a stream
 def read_stream(stream: subprocess.PIPE) -> None:
@@ -95,6 +98,7 @@ def container_exists_with(name: str, port: int) -> bool:
 
 
 def get_node_type(node_type: Optional[Union[str, NodeType]]) -> Optional[NodeType]:
+    NodeType = ImportFromSyft.import_node_type()
     if node_type is None:
         node_type = os.environ.get("ORCHESTRA_NODE_TYPE", NodeType.DOMAIN)
     try:
@@ -236,6 +240,7 @@ def deploy_to_python(
     queue_port: Optional[int] = None,
 ) -> Optional[NodeHandle]:
     stage_protocol_changes = ImportFromSyft.import_stage_protocol_changes()
+    NodeType = ImportFromSyft.import_node_type()
     sy = get_syft_client()
     if sy is None:
         return sy
@@ -259,7 +264,7 @@ def deploy_to_python(
         "processes": processes,
         "dev_mode": dev_mode,
         "tail": tail,
-        "node_type": str(node_type_enum),
+        "node_type": node_type_enum,
         "node_side_type": node_side_type,
         "enable_warnings": enable_warnings,
         # new kwargs
@@ -482,6 +487,7 @@ def launch(
         queue_port: Optional[int] = None,
         in_memory_workers: bool = True,
     ) -> Optional[NodeHandle]:
+        NodeType = ImportFromSyft.import_node_type()
         if dev_mode is True:
             os.environ["DEV_MODE"] = "True"
             thread_workers = True
@@ -495,8 +501,8 @@ def launch(
         dev_mode = str_to_bool(os.environ.get("DEV_MODE", f"{dev_mode}"))
 
         node_type_enum: Optional[NodeType] = get_node_type(node_type=node_type)
-        if not node_type_enum:
-            return None
+        # if not node_type_enum:
+        #     return None
 
         node_side_type_enum = (
             NodeSideType.HIGH_SIDE
diff --git a/packages/hagrid/hagrid/util.py b/packages/hagrid/hagrid/util.py
index a43a25e9689..41c5dcb39a5 100644
--- a/packages/hagrid/hagrid/util.py
+++ b/packages/hagrid/hagrid/util.py
@@ -22,17 +22,6 @@ def __str__(self) -> str:
         return self.value
 
 
-class NodeType(str, Enum):
-    DOMAIN = "domain"
-    NETWORK = "network"
-    ENCLAVE = "enclave"
-    GATEWAY = "gateway"
-
-    def __str__(self) -> str:
-        # Use values when transforming NodeType to str
-        return self.value
-
-
 class ImportFromSyft:
     @staticmethod
     def import_syft_error() -> Callable:
@@ -56,6 +45,16 @@ def stage_protocol_changes(*args: Any, **kwargs: Any) -> None:
 
         return stage_protocol_changes
 
+    @staticmethod
+    def import_node_type() -> Callable:
+        try:
+            # syft absolute
+            from syft.abstract_node import NodeType
+        except Exception:
+            NodeType = DummyNum
+
+        return NodeType
+
 
 def from_url(url: str) -> Tuple[str, str, int, str, Union[Any, str]]:
     try:

From 2c0a20dbcf6629306df12e1b0ced41a28c83a06a Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:02:13 +0530
Subject: [PATCH 13/28] fixed node type string errors in previous syft versions

---
 packages/hagrid/hagrid/orchestra.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py
index 4ff0d26293b..782f3a438c3 100644
--- a/packages/hagrid/hagrid/orchestra.py
+++ b/packages/hagrid/hagrid/orchestra.py
@@ -501,8 +501,6 @@ def launch(
         dev_mode = str_to_bool(os.environ.get("DEV_MODE", f"{dev_mode}"))
 
         node_type_enum: Optional[NodeType] = get_node_type(node_type=node_type)
-        # if not node_type_enum:
-        #     return None
 
         node_side_type_enum = (
             NodeSideType.HIGH_SIDE

From 82dcafc8228b877fdce56eb310a2834951f466f4 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:11:59 +0530
Subject: [PATCH 14/28] add hagrid test workflow

---
 .github/workflows/pr-tests-hagrid.yml | 59 +++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index ef70aa042df..59b2eb07ba2 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -106,3 +106,62 @@ jobs:
         if: steps.changes.outputs.hagrid == 'true'
         run: |
           twine check dist/*.whl
+
+  pr-tests-syft-hagrid-comptability:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+        syft-version: ["0.8.2", "0.8.2b6", "0.8.3"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.hagrid == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        if: steps.changes.outputs.hagrid == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.hagrid == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('packages/syft/setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('packages/syft/setup.cfg') }}
+
+      - name: Install Syft ${{ matrix.syft-version }}
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          pip install syft==${{ matrix.syft-version }}
+
+      - name: Run Orchestra Command
+        if: steps.changes.outputs.hagrid == 'true'
+        run: |
+          import syft as sy
+          domain-1 = sy.orchestra.launch(name="test-domain-1", port=8081, dev_mode=True, reset=True)
+          domain-2 = sy.orchestra.launch(name="test-domain-2", port=8082, dev_mode=False, reset=True)

From b53bbf0dd651fa510d47f21b4852d9f77e8d8512 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:14:05 +0530
Subject: [PATCH 15/28] updated workflow to install current hagrid

---
 .github/workflows/pr-tests-hagrid.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index 59b2eb07ba2..403156bc2a3 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -158,6 +158,7 @@ jobs:
         if: steps.changes.outputs.hagrid == 'true'
         run: |
           pip install syft==${{ matrix.syft-version }}
+          pip install -e packages/hagrid
 
       - name: Run Orchestra Command
         if: steps.changes.outputs.hagrid == 'true'

From 42027ab3cd95b8ba09fdcab17cff7138093b1cef Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:23:54 +0530
Subject: [PATCH 16/28] minor syft change to trigger all the notebook tests in
 syft

---
 packages/syft/src/syft/service/project/project.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py
index b550193b59d..57fc514c5d3 100644
--- a/packages/syft/src/syft/service/project/project.py
+++ b/packages/syft/src/syft/service/project/project.py
@@ -358,7 +358,7 @@ def poll_creation_wizard() -> List[Any]:
 Participants can then select the answer that best represents their opinion or preference"""
 
     description3 = """Since you didn't pass in questions, choices into .create_poll() (or you did so incorrectly),
-this wizard is going to guide you through the process of creating a poll."""
+this wizard is going to guide you through the process of creating a poll"""
 
     description4 = """In this wizard, we're going to ask you for a question and list of choices
 to create the poll. The Questions and choices are converted to strings"""

From 235e3c0ac6349bbec90846ffe4a5f5d4e084376b Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:25:25 +0530
Subject: [PATCH 17/28] fixed hagrid editable mode

---
 .github/workflows/pr-tests-hagrid.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index 403156bc2a3..bcf6672087c 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -158,7 +158,7 @@ jobs:
         if: steps.changes.outputs.hagrid == 'true'
         run: |
           pip install syft==${{ matrix.syft-version }}
-          pip install -e packages/hagrid
+          pip install packages/hagrid
 
       - name: Run Orchestra Command
         if: steps.changes.outputs.hagrid == 'true'

From 85310d46c52e4e2a4658cddcdf65e602451c96b8 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:29:27 +0530
Subject: [PATCH 18/28] added pwd command

---
 .github/workflows/pr-tests-hagrid.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index bcf6672087c..b1ebb535ad3 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -157,6 +157,7 @@ jobs:
       - name: Install Syft ${{ matrix.syft-version }}
         if: steps.changes.outputs.hagrid == 'true'
         run: |
+          pwd
           pip install syft==${{ matrix.syft-version }}
           pip install packages/hagrid
 

From 90098e3e31da4c79606c3bf3867a408aa6e7c95c Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:32:59 +0530
Subject: [PATCH 19/28] changed hagrid installation paths

---
 .github/workflows/pr-tests-hagrid.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index b1ebb535ad3..a4b5e951e6c 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -157,9 +157,8 @@ jobs:
       - name: Install Syft ${{ matrix.syft-version }}
         if: steps.changes.outputs.hagrid == 'true'
         run: |
-          pwd
           pip install syft==${{ matrix.syft-version }}
-          pip install packages/hagrid
+          pip install .
 
       - name: Run Orchestra Command
         if: steps.changes.outputs.hagrid == 'true'

From fc6892140e511f9a63864f608c5b5397146386b7 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:39:03 +0530
Subject: [PATCH 20/28] switched to python interpreter for shell execution

---
 .github/workflows/pr-tests-hagrid.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index a4b5e951e6c..12462d89099 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -163,6 +163,5 @@ jobs:
       - name: Run Orchestra Command
         if: steps.changes.outputs.hagrid == 'true'
         run: |
-          import syft as sy
-          domain-1 = sy.orchestra.launch(name="test-domain-1", port=8081, dev_mode=True, reset=True)
-          domain-2 = sy.orchestra.launch(name="test-domain-2", port=8082, dev_mode=False, reset=True)
+          python -c "import syft as sy; domain-1 = sy.orchestra.launch(name='test-domain-1', port=8081, dev_mode=True, reset=True)"
+          python -c "import syft as sy; domain-2 = sy.orchestra.launch(name='test-domain-2', port=8082, dev_mode=False, reset=True)"

From 0c2ece035f72aae03c856573736a96175791c9b8 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:41:32 +0530
Subject: [PATCH 21/28] changed variable naming

---
 .github/workflows/pr-tests-hagrid.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index 12462d89099..13681dbea60 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -163,5 +163,5 @@ jobs:
       - name: Run Orchestra Command
         if: steps.changes.outputs.hagrid == 'true'
         run: |
-          python -c "import syft as sy; domain-1 = sy.orchestra.launch(name='test-domain-1', port=8081, dev_mode=True, reset=True)"
-          python -c "import syft as sy; domain-2 = sy.orchestra.launch(name='test-domain-2', port=8082, dev_mode=False, reset=True)"
+          python -c "import syft as sy; domain1 = sy.orchestra.launch(name='test-domain-1', port=8081, dev_mode=True, reset=True)"
+          python -c "import syft as sy; domain2 = sy.orchestra.launch(name='test-domain-2', port=8082, dev_mode=False, reset=True)"

From 4337d803182f17043dd86aadbe97823fd5cb0065 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 17:01:40 +0530
Subject: [PATCH 22/28] pin ml-dyptes version fix hagrid versioning

---
 .github/workflows/cd-post-release-tests.yml | 351 ++++++++++++++++++++
 .github/workflows/pr-tests-hagrid.yml       |   7 +-
 2 files changed, 356 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/cd-post-release-tests.yml

diff --git a/.github/workflows/cd-post-release-tests.yml b/.github/workflows/cd-post-release-tests.yml
new file mode 100644
index 00000000000..ff682991899
--- /dev/null
+++ b/.github/workflows/cd-post-release-tests.yml
@@ -0,0 +1,351 @@
+name: CD - Post Release Tests
+
+on:
+  workflow_call:
+
+  workflow_dispatch:
+    inputs:
+      none:
+        description: "Run Tests Manually"
+        required: false
+
+concurrency:
+  group: syft-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }}
+  cancel-in-progress: true
+
+jobs:
+  pr-tests-syft-unit:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.11"]
+        include:
+          - python-version: "3.9"
+            os: "ubuntu-latest"
+          - python-version: "3.10"
+            os: "ubuntu-latest"
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      # - name: Permission to home directory
+      #   if: matrix.os == 'ubuntu-latest'
+      #   run: |
+      #     sudo chown -R $USER:$USER $HOME
+      - name: "clean .git/config"
+        if: matrix.os == 'windows'
+        continue-on-error: true
+        shell: bash
+        run: |
+          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
+          rm ${GITHUB_WORKSPACE}/.git/config
+
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.syft == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        if: steps.changes.outputs.syft == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.syft == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install Dependencies
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          pip install --upgrade tox packaging wheel --default-timeout=60
+
+      - name: Docker on MacOS
+        if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-latest'
+        uses: crazy-max/ghaction-setup-docker@v3.0.0
+
+      - name: Run unit tests
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          tox -e syft.test.unit
+
+  pr-tests-syft-notebook-python:
+    strategy:
+      max-parallel: 99
+      matrix:
+        # Disable on windows until its flakyness is reduced.
+        # os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.11"]
+        deployment-type: ["python"]
+        notebook-paths: ["tutorials"]
+        include:
+          - python-version: "3.9"
+            os: "ubuntu-latest"
+            deployment-type: "python"
+            notebook-paths: "tutorials"
+          - python-version: "3.10"
+            os: "ubuntu-latest"
+            deployment-type: "python"
+            notebook-paths: "tutorials"
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      # - name: Permission to home directory
+      #   if: matrix.os == 'ubuntu-latest'
+      #   run: |
+      #     sudo chown -R $USER:$USER $HOME
+      - name: "clean .git/config"
+        if: matrix.os == 'windows'
+        continue-on-error: true
+        shell: bash
+        run: |
+          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
+          rm ${GITHUB_WORKSPACE}/.git/config
+
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install Dependencies
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
+        run: |
+          pip install --upgrade tox packaging wheel --default-timeout=60
+
+      - name: Run notebook tests
+        uses: nick-fields/retry@v2
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
+        env:
+          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
+          TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}"
+        with:
+          timeout_seconds: 2400
+          max_attempts: 3
+          command: tox -e syft.test.notebook
+
+  pr-tests-syft-notebook-container:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.9", "3.10", "3.11"]
+        deployment-type: ["single_container"]
+        notebook-paths: ["api/0.8"]
+      fail-fast: false
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+      # - name: Permission to home directory
+      #   if: matrix.os == 'ubuntu-latest'
+      #   run: |
+      #     sudo chown -R $USER:$USER $HOME
+      - uses: actions/checkout@v4
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install Dependencies
+        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
+        run: |
+          pip install --upgrade tox packaging wheel --default-timeout=60
+
+      - name: Docker Compose on Linux
+        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'ubuntu-latest'
+        shell: bash
+        run: |
+          mkdir -p ~/.docker/cli-plugins
+          DOCKER_COMPOSE_VERSION=v2.21.0
+          curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
+          chmod +x ~/.docker/cli-plugins/docker-compose
+          docker compose version
+
+      - name: Docker on MacOS
+        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest'
+        uses: crazy-max/ghaction-setup-docker@v3.0.0
+
+      - name: Docker Compose on MacOS
+        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest'
+        shell: bash
+        run: |
+          brew install docker-compose
+          mkdir -p ~/.docker/cli-plugins
+          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose || true
+          docker compose version
+
+      - name: Docker on Windows
+        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'windows-latest'
+        shell: pwsh
+        run: |
+          [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+          Start-BitsTransfer -Source "https://download.docker.com/win/static/stable/x86_64/docker-23.0.1.zip"
+          Expand-Archive docker-23.0.1.zip -DestinationPath $Env:ProgramFiles
+          &$Env:ProgramFiles\Docker\dockerd --register-service
+          Start-Service docker
+          docker version
+          docker compose version
+
+      - name: Run unit tests
+        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
+        env:
+          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
+          DEV_MODE: "True" # force orchestra --build
+          TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}"
+        run: |
+          tox -e stack.test.notebook
+
+  pr-tests-syft-security:
+    strategy:
+      max-parallel: 1
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      # - name: Permission to home directory
+      #   if: matrix.os == 'ubuntu-latest'
+      #   run: |
+      #     sudo chown -R $USER:$USER $HOME
+      - uses: actions/checkout@v4
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.syft == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        if: steps.changes.outputs.syft == 'true'
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.syft == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install Dependencies
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          pip install --upgrade tox packaging wheel --default-timeout=60
+
+      - name: Scan for security issues
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          tox -e syft.test.security
diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index 13681dbea60..63dbff22dfc 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -157,11 +157,14 @@ jobs:
       - name: Install Syft ${{ matrix.syft-version }}
         if: steps.changes.outputs.hagrid == 'true'
         run: |
+          https://github.com/google/jax/issues/17693
+          pip install ml-dtypes==0.2.0
+          # Due to jax version<=0.4.10
           pip install syft==${{ matrix.syft-version }}
           pip install .
 
       - name: Run Orchestra Command
         if: steps.changes.outputs.hagrid == 'true'
         run: |
-          python -c "import syft as sy; domain1 = sy.orchestra.launch(name='test-domain-1', port=8081, dev_mode=True, reset=True)"
-          python -c "import syft as sy; domain2 = sy.orchestra.launch(name='test-domain-2', port=8082, dev_mode=False, reset=True)"
+          python -c "import syft as sy; domain1 = sy.orchestra.launch(name='test-domain-1', dev_mode=True, reset=True)"
+          python -c "import syft as sy; domain2 = sy.orchestra.launch(name='test-domain-2',dev_mode=False, reset=True)"

From 86518d834b2f2a74159b6bab0bafb8366fc5bf43 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 17:04:19 +0530
Subject: [PATCH 23/28] minor comment fix

---
 .github/workflows/pr-tests-hagrid.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
index 63dbff22dfc..b8dd7e1da2a 100644
--- a/.github/workflows/pr-tests-hagrid.yml
+++ b/.github/workflows/pr-tests-hagrid.yml
@@ -154,12 +154,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('packages/syft/setup.cfg') }}
 
+      # https://github.com/google/jax/issues/17693
+      # pinning ml-dtypes due to jax version==0.4.10
       - name: Install Syft ${{ matrix.syft-version }}
         if: steps.changes.outputs.hagrid == 'true'
         run: |
-          https://github.com/google/jax/issues/17693
           pip install ml-dtypes==0.2.0
-          # Due to jax version<=0.4.10
           pip install syft==${{ matrix.syft-version }}
           pip install .
 

From 509591e43a33199f6c9bc8fcab3e97e4af956db5 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 17:16:10 +0530
Subject: [PATCH 24/28] modified orchestra migrate function

---
 .github/workflows/cd-post-release-tests.yml | 316 +-------------------
 packages/hagrid/hagrid/orchestra.py         |   2 +-
 2 files changed, 11 insertions(+), 307 deletions(-)

diff --git a/.github/workflows/cd-post-release-tests.yml b/.github/workflows/cd-post-release-tests.yml
index ff682991899..6996b5a06df 100644
--- a/.github/workflows/cd-post-release-tests.yml
+++ b/.github/workflows/cd-post-release-tests.yml
@@ -5,327 +5,34 @@ on:
 
   workflow_dispatch:
     inputs:
-      none:
-        description: "Run Tests Manually"
-        required: false
-
-concurrency:
-  group: syft-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }}
-  cancel-in-progress: true
+      syft_version:
+        description: "Syft version to test"
+        required: true
+        type: string
 
 jobs:
-  pr-tests-syft-unit:
+  syft-install-check:
     strategy:
       max-parallel: 99
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.11"]
-        include:
-          - python-version: "3.9"
-            os: "ubuntu-latest"
-          - python-version: "3.10"
-            os: "ubuntu-latest"
+        python-version: ["3.11", "3.10", "3.9"]
 
     runs-on: ${{ matrix.os }}
     steps:
-      # - name: Permission to home directory
-      #   if: matrix.os == 'ubuntu-latest'
-      #   run: |
-      #     sudo chown -R $USER:$USER $HOME
-      - name: "clean .git/config"
-        if: matrix.os == 'windows'
-        continue-on-error: true
-        shell: bash
-        run: |
-          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
-          rm ${GITHUB_WORKSPACE}/.git/config
-
       - uses: actions/checkout@v4
 
-      - name: Check for file changes
-        uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          base: ${{ github.ref }}
-          token: ${{ github.token }}
-          filters: .github/file-filters.yml
-
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
-        if: steps.changes.outputs.syft == 'true'
-        with:
-          python-version: ${{ matrix.python-version }}
 
-      - name: Upgrade pip
-        if: steps.changes.outputs.syft == 'true'
-        run: |
-          python -m pip install --upgrade --user pip
-
-      - name: Get pip cache dir
-        id: pip-cache
-        if: steps.changes.outputs.syft == 'true'
-        shell: bash
-        run: |
-          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-
-      - name: pip cache
-        uses: actions/cache@v4
-        if: steps.changes.outputs.syft == 'true'
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
-
-      - name: Install Dependencies
-        if: steps.changes.outputs.syft == 'true'
-        run: |
-          pip install --upgrade tox packaging wheel --default-timeout=60
-
-      - name: Docker on MacOS
-        if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-latest'
-        uses: crazy-max/ghaction-setup-docker@v3.0.0
-
-      - name: Run unit tests
-        if: steps.changes.outputs.syft == 'true'
-        run: |
-          tox -e syft.test.unit
-
-  pr-tests-syft-notebook-python:
-    strategy:
-      max-parallel: 99
-      matrix:
-        # Disable on windows until its flakyness is reduced.
-        # os: [ubuntu-latest, macos-latest, windows-latest]
-        os: [ubuntu-latest, macos-latest]
-        python-version: ["3.11"]
-        deployment-type: ["python"]
-        notebook-paths: ["tutorials"]
-        include:
-          - python-version: "3.9"
-            os: "ubuntu-latest"
-            deployment-type: "python"
-            notebook-paths: "tutorials"
-          - python-version: "3.10"
-            os: "ubuntu-latest"
-            deployment-type: "python"
-            notebook-paths: "tutorials"
-
-    runs-on: ${{ matrix.os }}
-    steps:
-      # - name: Permission to home directory
-      #   if: matrix.os == 'ubuntu-latest'
-      #   run: |
-      #     sudo chown -R $USER:$USER $HOME
-      - name: "clean .git/config"
-        if: matrix.os == 'windows'
-        continue-on-error: true
-        shell: bash
-        run: |
-          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
-          rm ${GITHUB_WORKSPACE}/.git/config
-
-      - uses: actions/checkout@v4
-
-      - name: Check for file changes
-        uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          base: ${{ github.ref }}
-          token: ${{ github.token }}
-          filters: .github/file-filters.yml
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Upgrade pip
-        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
-        run: |
-          python -m pip install --upgrade --user pip
-
-      - name: Get pip cache dir
-        id: pip-cache
-        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
-        shell: bash
-        run: |
-          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-
-      - name: pip cache
-        uses: actions/cache@v4
-        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
-
-      - name: Install Dependencies
-        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
-        run: |
-          pip install --upgrade tox packaging wheel --default-timeout=60
-
-      - name: Run notebook tests
-        uses: nick-fields/retry@v2
-        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
-        env:
-          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
-          TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}"
-        with:
-          timeout_seconds: 2400
-          max_attempts: 3
-          command: tox -e syft.test.notebook
-
-  pr-tests-syft-notebook-container:
-    strategy:
-      max-parallel: 99
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11"]
-        deployment-type: ["single_container"]
-        notebook-paths: ["api/0.8"]
-      fail-fast: false
-
-    runs-on: ${{matrix.os}}
-
-    steps:
-      # - name: Permission to home directory
-      #   if: matrix.os == 'ubuntu-latest'
-      #   run: |
-      #     sudo chown -R $USER:$USER $HOME
-      - uses: actions/checkout@v4
-
-      # free 10GB of space
-      - name: Remove unnecessary files
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          docker image prune --all --force
-          docker builder prune --all --force
-          docker system prune --all --force
-
-      - name: Check for file changes
-        uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          base: ${{ github.ref }}
-          token: ${{ github.token }}
-          filters: .github/file-filters.yml
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Upgrade pip
-        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
-        run: |
-          python -m pip install --upgrade --user pip
-
-      - name: Get pip cache dir
-        id: pip-cache
-        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
-        shell: bash
-        run: |
-          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-
-      - name: pip cache
-        uses: actions/cache@v4
-        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
-
-      - name: Install Dependencies
-        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
-        run: |
-          pip install --upgrade tox packaging wheel --default-timeout=60
-
-      - name: Docker Compose on Linux
-        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'ubuntu-latest'
-        shell: bash
-        run: |
-          mkdir -p ~/.docker/cli-plugins
-          DOCKER_COMPOSE_VERSION=v2.21.0
-          curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
-          chmod +x ~/.docker/cli-plugins/docker-compose
-          docker compose version
-
-      - name: Docker on MacOS
-        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest'
-        uses: crazy-max/ghaction-setup-docker@v3.0.0
-
-      - name: Docker Compose on MacOS
-        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest'
-        shell: bash
-        run: |
-          brew install docker-compose
-          mkdir -p ~/.docker/cli-plugins
-          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose || true
-          docker compose version
-
-      - name: Docker on Windows
-        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'windows-latest'
-        shell: pwsh
-        run: |
-          [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
-          Start-BitsTransfer -Source "https://download.docker.com/win/static/stable/x86_64/docker-23.0.1.zip"
-          Expand-Archive docker-23.0.1.zip -DestinationPath $Env:ProgramFiles
-          &$Env:ProgramFiles\Docker\dockerd --register-service
-          Start-Service docker
-          docker version
-          docker compose version
-
-      - name: Run unit tests
-        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
-        env:
-          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
-          DEV_MODE: "True" # force orchestra --build
-          TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}"
-        run: |
-          tox -e stack.test.notebook
-
-  pr-tests-syft-security:
-    strategy:
-      max-parallel: 1
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.11"]
-
-    runs-on: ${{ matrix.os }}
-    steps:
-      # - name: Permission to home directory
-      #   if: matrix.os == 'ubuntu-latest'
-      #   run: |
-      #     sudo chown -R $USER:$USER $HOME
-      - uses: actions/checkout@v4
-      - name: Check for file changes
-        uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          base: ${{ github.ref }}
-          token: ${{ github.token }}
-          filters: .github/file-filters.yml
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        if: steps.changes.outputs.syft == 'true'
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Upgrade pip
-        if: steps.changes.outputs.syft == 'true'
         run: |
           python -m pip install --upgrade --user pip
 
       - name: Get pip cache dir
-        if: steps.changes.outputs.syft == 'true'
         id: pip-cache
         shell: bash
         run: |
@@ -333,19 +40,16 @@ jobs:
 
       - name: pip cache
         uses: actions/cache@v4
-        if: steps.changes.outputs.syft == 'true'
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
           key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
           restore-keys: |
             ${{ runner.os }}-pip-py${{ matrix.python-version }}-
 
-      - name: Install Dependencies
-        if: steps.changes.outputs.syft == 'true'
+      - name: Install Syft
         run: |
-          pip install --upgrade tox packaging wheel --default-timeout=60
+          pip install syft==${{ github.event.inputs.syft_version }}
 
-      - name: Scan for security issues
-        if: steps.changes.outputs.syft == 'true'
+      - name: Check Syft version
         run: |
-          tox -e syft.test.security
+          python -c "import syft; print(syft.__version__)"
diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py
index 782f3a438c3..41f2656f35f 100644
--- a/packages/hagrid/hagrid/orchestra.py
+++ b/packages/hagrid/hagrid/orchestra.py
@@ -301,7 +301,7 @@ def deploy_to_python(
             worker_class = worker_classes[node_type_enum]
             sig = inspect.signature(worker_class.named)
             supported_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters}
-            if "node_type" in sig.parameters.keys():
+            if "node_type" in sig.parameters.keys() and "migration" in sig.parameters:
                 supported_kwargs["migrate"] = True
             worker = worker_class.named(**supported_kwargs)
         else:

From 397a35c2bfad98e6da4c8fb85fde880d0e78e1ce Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 17:17:43 +0530
Subject: [PATCH 25/28] modified orchestra migrate function -2

---
 packages/hagrid/hagrid/orchestra.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py
index 41f2656f35f..8a0e74c06b6 100644
--- a/packages/hagrid/hagrid/orchestra.py
+++ b/packages/hagrid/hagrid/orchestra.py
@@ -301,7 +301,7 @@ def deploy_to_python(
             worker_class = worker_classes[node_type_enum]
             sig = inspect.signature(worker_class.named)
             supported_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters}
-            if "node_type" in sig.parameters.keys() and "migration" in sig.parameters:
+            if "node_type" in sig.parameters.keys() and "migrate" in sig.parameters:
                 supported_kwargs["migrate"] = True
             worker = worker_class.named(**supported_kwargs)
         else:

From 1eb1e0d34c62b44cf99ab42f43afbce57466dc34 Mon Sep 17 00:00:00 2001
From: alfred-openmined-bot
 <145415986+alfred-openmined-bot@users.noreply.github.com>
Date: Wed, 14 Feb 2024 12:18:19 +0000
Subject: [PATCH 26/28] [hagrid] bump version

---
 packages/hagrid/.bumpversion.cfg             | 2 +-
 packages/hagrid/hagrid/manifest_template.yml | 4 ++--
 packages/hagrid/hagrid/version.py            | 2 +-
 packages/hagrid/setup.py                     | 2 +-
 scripts/hagrid_hash                          | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/hagrid/.bumpversion.cfg b/packages/hagrid/.bumpversion.cfg
index 7c93af26e37..f43bf0b3f46 100644
--- a/packages/hagrid/.bumpversion.cfg
+++ b/packages/hagrid/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.108
+current_version = 0.3.109
 tag = False
 tag_name = {new_version}
 commit = True
diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml
index f67bb5d5bbd..af9dfeeeee5 100644
--- a/packages/hagrid/hagrid/manifest_template.yml
+++ b/packages/hagrid/hagrid/manifest_template.yml
@@ -1,9 +1,9 @@
 manifestVersion: 0.1
-hagrid_version: 0.3.108
+hagrid_version: 0.3.109
 syft_version: 0.8.4-beta.22
 dockerTag: 0.8.4-beta.22
 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/
-hash: 799965e620b01f9cd1d8e7f23cc4cdc0fbc0410b
+hash: 035c668889ceebb2d1334a2a0f14996a25c3203e
 target_dir: ~/.hagrid/PySyft/
 files:
   grid:
diff --git a/packages/hagrid/hagrid/version.py b/packages/hagrid/hagrid/version.py
index 19175a1670c..a2feed2c05c 100644
--- a/packages/hagrid/hagrid/version.py
+++ b/packages/hagrid/hagrid/version.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # HAGrid Version
-__version__ = "0.3.108"
+__version__ = "0.3.109"
 
 if __name__ == "__main__":
     print(__version__)
diff --git a/packages/hagrid/setup.py b/packages/hagrid/setup.py
index 94bc8d4c77b..54adf776cba 100644
--- a/packages/hagrid/setup.py
+++ b/packages/hagrid/setup.py
@@ -5,7 +5,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = "0.3.108"
+__version__ = "0.3.109"
 
 DATA_FILES = {"img": ["hagrid/img/*.png"], "hagrid": ["*.yml"]}
 
diff --git a/scripts/hagrid_hash b/scripts/hagrid_hash
index 010a1364538..4683975b16b 100644
--- a/scripts/hagrid_hash
+++ b/scripts/hagrid_hash
@@ -1 +1 @@
-7a5b3ec4541268b9b6d41ae76a771231
+d1010f030826e9eae7fbda9daa9b2dcb

From c9c11eb98a6f01e5d00a173798020faf25fb4f7e Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Wed, 14 Feb 2024 17:51:17 +0530
Subject: [PATCH 27/28] fixed concurrency in hagrid releases

---
 .github/workflows/cd-hagrid.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/cd-hagrid.yml b/.github/workflows/cd-hagrid.yml
index 817e2c43702..a349e12b94a 100644
--- a/.github/workflows/cd-hagrid.yml
+++ b/.github/workflows/cd-hagrid.yml
@@ -11,6 +11,12 @@ on:
         required: false
         default: "false"
 
+# Prevents concurrent runs of the same workflow
+# while the previous run is still in progress
+concurrency:
+  group: "CD - Hagrid"
+  cancel-in-progress: false
+
 jobs:
   call-pr-tests-linting:
     if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks

From a0f011e6ce2853bd61ef4a00bb284b0899c141ce Mon Sep 17 00:00:00 2001
From: alfred-openmined-bot
 <145415986+alfred-openmined-bot@users.noreply.github.com>
Date: Wed, 14 Feb 2024 12:25:56 +0000
Subject: [PATCH 28/28] [hagrid] bump version

---
 packages/hagrid/.bumpversion.cfg             | 2 +-
 packages/hagrid/hagrid/manifest_template.yml | 4 ++--
 packages/hagrid/hagrid/version.py            | 2 +-
 packages/hagrid/setup.py                     | 2 +-
 scripts/hagrid_hash                          | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/hagrid/.bumpversion.cfg b/packages/hagrid/.bumpversion.cfg
index f43bf0b3f46..e603df6ab55 100644
--- a/packages/hagrid/.bumpversion.cfg
+++ b/packages/hagrid/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.109
+current_version = 0.3.110
 tag = False
 tag_name = {new_version}
 commit = True
diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml
index af9dfeeeee5..4291013ce5e 100644
--- a/packages/hagrid/hagrid/manifest_template.yml
+++ b/packages/hagrid/hagrid/manifest_template.yml
@@ -1,9 +1,9 @@
 manifestVersion: 0.1
-hagrid_version: 0.3.109
+hagrid_version: 0.3.110
 syft_version: 0.8.4-beta.22
 dockerTag: 0.8.4-beta.22
 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/
-hash: 035c668889ceebb2d1334a2a0f14996a25c3203e
+hash: 7e36da6aac7dc6b63c8f23ad3d6d07cf51281362
 target_dir: ~/.hagrid/PySyft/
 files:
   grid:
diff --git a/packages/hagrid/hagrid/version.py b/packages/hagrid/hagrid/version.py
index a2feed2c05c..84581fc53e7 100644
--- a/packages/hagrid/hagrid/version.py
+++ b/packages/hagrid/hagrid/version.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # HAGrid Version
-__version__ = "0.3.109"
+__version__ = "0.3.110"
 
 if __name__ == "__main__":
     print(__version__)
diff --git a/packages/hagrid/setup.py b/packages/hagrid/setup.py
index 54adf776cba..a3e560ce20a 100644
--- a/packages/hagrid/setup.py
+++ b/packages/hagrid/setup.py
@@ -5,7 +5,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = "0.3.109"
+__version__ = "0.3.110"
 
 DATA_FILES = {"img": ["hagrid/img/*.png"], "hagrid": ["*.yml"]}
 
diff --git a/scripts/hagrid_hash b/scripts/hagrid_hash
index 4683975b16b..2791d55f62f 100644
--- a/scripts/hagrid_hash
+++ b/scripts/hagrid_hash
@@ -1 +1 @@
-d1010f030826e9eae7fbda9daa9b2dcb
+8062e2de4754fbc983daabd67ca5d8cc