From b807c551843134bb7c414f76fba32b687fb40227 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 07:29:21 +0200 Subject: [PATCH 01/83] Update api.py --- packages/syft/src/syft/client/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index c40677221e2..7d119b3d20b 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -740,7 +740,7 @@ def debox_signed_syftapicall_response( def downgrade_signature(signature: Signature, object_versions: dict) -> Signature: migrated_parameters = [] - for _, parameter in signature.parameters.items(): + for parameter in signature.parameters.values(): annotation = unwrap_and_migrate_annotation( parameter.annotation, object_versions ) From d45e531e4d6f44607fd4a967a392d34662851815 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 08:10:48 +0200 Subject: [PATCH 02/83] Update api.py --- packages/syft/src/syft/client/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index 7d119b3d20b..4efdcbf6469 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -1101,7 +1101,7 @@ def build_endpoint_tree( endpoints: dict[str, LibEndpoint], communication_protocol: PROTOCOL_TYPE ) -> APIModule: api_module = APIModule(path="", refresh_callback=self.refresh_api_callback) - for _, v in endpoints.items(): + for v in endpoints.values(): signature = v.signature if not v.has_self: signature = signature_remove_self(signature) From 806836244f19ed2650cc5c7fb9bc3fbf258a367f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 08:16:32 +0200 Subject: [PATCH 03/83] Update registry.py --- packages/syft/src/syft/client/registry.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/syft/src/syft/client/registry.py b/packages/syft/src/syft/client/registry.py index ee57b642f53..4128af452d8 100644 --- a/packages/syft/src/syft/client/registry.py +++ b/packages/syft/src/syft/client/registry.py @@ -405,10 +405,7 @@ def check_enclave(enclave: dict) -> dict[Any, Any] | None: executor.map(lambda enclave: check_enclave(enclave), enclaves) ) - online_enclaves = [] - for each in _online_enclaves: - if each is not None: - online_enclaves.append(each) + online_enclaves = [each for each in _online_enclaves if each is not None] return online_enclaves def _repr_html_(self) -> str: From 21b53f4956791801c3e092f9c2e9c8e48e600551 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 08:30:49 +0200 Subject: [PATCH 04/83] Update node.py --- packages/syft/src/syft/node/node.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 1e2c00c6f24..65a3ed80eda 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -619,7 +619,7 @@ def add_consumer_for_service( consumer.run() def remove_consumer_with_id(self, syft_worker_id: UID) -> None: - for _, consumers in self.queue_manager.consumers.items(): + for consumers in self.queue_manager.consumers.values(): # Grab the list of consumers for the given queue consumer_to_pop = None for consumer_idx, consumer in enumerate(consumers): @@ -834,8 +834,7 @@ def __repr__(self) -> str: service_string = "" if not self.is_subprocess: services = [] - for service in self.services: - services.append(service.__name__) + services = [service.__name__ for service in self.services] service_string = ", ".join(sorted(services)) service_string = f"\n\nServices:\n{service_string}" return f"{type(self).__name__}: {self.name} - {self.id} - {self.node_type}{service_string}" From b255bf463f60078555360055d4448eb0d892626e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 08:51:50 +0200 Subject: [PATCH 05/83] Update k8s.py --- packages/syft/src/syft/custom_worker/k8s.py | 23 ++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index cb4b5765e62..21eea90146e 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -120,14 +120,12 @@ def resolve_pod(client: kr8s.Api, pod: str | Pod) -> Pod | None: @staticmethod def get_logs(pods: list[Pod]) -> str: - """Combine and return logs for all the pods as string""" - logs = [] - for pod in pods: - logs.append(f"----------Logs for pod={pod.metadata.name}----------") - for log in pod.logs(): - logs.append(log) + """Combine and return logs for all the pods as a single string.""" + return "\n".join( + f"----------Logs for pod={pod.metadata.name}----------\n{''.join(pod.logs())}" + for pod in pods + ) - return "\n".join(logs) @staticmethod def get_pod_status(pod: Pod) -> PodStatus | None: @@ -150,11 +148,12 @@ def get_pod_env(pod: Pod) -> list[dict] | None: @staticmethod def get_container_exit_code(pods: list[Pod]) -> list[int]: """Return the exit codes of all the containers in the given pods.""" - exit_codes = [] - for pod in pods: - for container_status in pod.status.containerStatuses: - exit_codes.append(container_status.state.terminated.exitCode) - return exit_codes + return [ + container_status.state.terminated.exitCode + for pod in pods + for container_status in pod.status.containerStatuses + ] + @staticmethod def get_container_exit_message(pods: list[Pod]) -> str | None: From f153c6982d7e9b3beb1289786c98d10f2e039c77 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 08:54:12 +0200 Subject: [PATCH 06/83] Update k8s.py --- packages/syft/src/syft/custom_worker/k8s.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index 21eea90146e..7f76cb94337 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -126,7 +126,6 @@ def get_logs(pods: list[Pod]) -> str: for pod in pods ) - @staticmethod def get_pod_status(pod: Pod) -> PodStatus | None: """Map the status of the given pod to PodStatuss.""" @@ -154,7 +153,6 @@ def get_container_exit_code(pods: list[Pod]) -> list[int]: for container_status in pod.status.containerStatuses ] - @staticmethod def get_container_exit_message(pods: list[Pod]) -> str | None: """Return the exit message of the first container that exited with non-zero code.""" From 0c286511136fa820bec7617f9af85c355dfc741f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 08:56:44 +0200 Subject: [PATCH 07/83] Update client.py --- packages/syft/src/syft/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 498c22e7536..72a048d2d2b 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -589,7 +589,7 @@ def get_nested_codes(code: UserCode) -> list[UserCode]: if code.nested_codes is None: return result - for _, (linked_code_obj, _) in code.nested_codes.items(): + for linked_code_obj in code.nested_codes.keys(): nested_code = linked_code_obj.resolve nested_code = deepcopy(nested_code) nested_code.node_uid = code.node_uid From bc13c418cdc2cc52d46b8b8ae5713bd26f186bdc Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:02:38 +0200 Subject: [PATCH 08/83] Update server.py --- packages/syft/src/syft/node/server.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/syft/src/syft/node/server.py b/packages/syft/src/syft/node/server.py index f5f05bf35ac..cf86da91d6a 100644 --- a/packages/syft/src/syft/node/server.py +++ b/packages/syft/src/syft/node/server.py @@ -285,14 +285,12 @@ def find_python_processes_on_port(port: int) -> list[int]: python_pids = [] for pid in pids: + if system == "Windows": + command = f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine" + else: + command = f"ps -p {pid} -o pid,command" + try: - if system == "Windows": - command = ( - f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine" - ) - else: - command = f"ps -p {pid} -o pid,command" - process = subprocess.Popen( # nosec command, shell=True, @@ -301,14 +299,14 @@ def find_python_processes_on_port(port: int) -> list[int]: text=True, ) output, _ = process.communicate() - lines = output.strip().split("\n") - - if len(lines) > 1 and "python" in lines[1].lower(): - python_pids.append(pid) - except Exception as e: print(f"Error checking process {pid}: {e}") - + continue + + lines = output.strip().split("\n") + if len(lines) > 1 and "python" in lines[1].lower(): + python_pids.append(pid) + return python_pids From 441e9266c0d7ce870c380979de79b442c56b79dd Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:05:20 +0200 Subject: [PATCH 09/83] Update server.py --- packages/syft/src/syft/node/server.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/node/server.py b/packages/syft/src/syft/node/server.py index cf86da91d6a..3da97e4b0a2 100644 --- a/packages/syft/src/syft/node/server.py +++ b/packages/syft/src/syft/node/server.py @@ -286,10 +286,12 @@ def find_python_processes_on_port(port: int) -> list[int]: python_pids = [] for pid in pids: if system == "Windows": - command = f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine" + command = ( + f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine" + ) else: command = f"ps -p {pid} -o pid,command" - + try: process = subprocess.Popen( # nosec command, @@ -302,11 +304,11 @@ def find_python_processes_on_port(port: int) -> list[int]: except Exception as e: print(f"Error checking process {pid}: {e}") continue - + lines = output.strip().split("\n") if len(lines) > 1 and "python" in lines[1].lower(): python_pids.append(pid) - + return python_pids From d96a4dcde1d870f52e05a406d46b9d19e1edb833 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:12:39 +0200 Subject: [PATCH 10/83] Update data_protocol.py --- packages/syft/src/syft/protocol/data_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py index 79f0d680658..f357f867467 100644 --- a/packages/syft/src/syft/protocol/data_protocol.py +++ b/packages/syft/src/syft/protocol/data_protocol.py @@ -507,7 +507,7 @@ def calculate_supported_protocols(self) -> dict: # we assume its supported until we prove otherwise protocol_supported[v] = True # iterate through each object - for canonical_name, _ in version_data["object_versions"].items(): + for canonical_name in version_data["object_versions"].keys(): if canonical_name not in self.state: protocol_supported[v] = False break From cbb80e396ed038419a4f549308aef355eb61af34 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:25:17 +0200 Subject: [PATCH 11/83] Update recursive.py --- packages/syft/src/syft/serde/recursive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/serde/recursive.py b/packages/syft/src/syft/serde/recursive.py index 02957e5f23d..f009ea34299 100644 --- a/packages/syft/src/syft/serde/recursive.py +++ b/packages/syft/src/syft/serde/recursive.py @@ -311,7 +311,7 @@ def rs_proto2object(proto: _DynamicStructBuilder) -> Any: # relative from ..node.node import CODE_RELOADER - for _, load_user_code in CODE_RELOADER.items(): + for load_user_code in CODE_RELOADER.values(): load_user_code() try: class_type = getattr(sys.modules[".".join(module_parts)], klass) From 0717723931943faf688fae800e10b951b803c666 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:30:17 +0200 Subject: [PATCH 12/83] Update recursive_primitives.py --- packages/syft/src/syft/serde/recursive_primitives.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/syft/src/syft/serde/recursive_primitives.py b/packages/syft/src/syft/serde/recursive_primitives.py index cb90932247a..a536950b5b7 100644 --- a/packages/syft/src/syft/serde/recursive_primitives.py +++ b/packages/syft/src/syft/serde/recursive_primitives.py @@ -64,17 +64,14 @@ def serialize_iterable(iterable: Collection) -> bytes: def deserialize_iterable(iterable_type: type, blob: bytes) -> Collection: - # relative from .deserialize import _deserialize MAX_TRAVERSAL_LIMIT = 2**64 - 1 - values = [] with iterable_schema.from_bytes( blob, traversal_limit_in_words=MAX_TRAVERSAL_LIMIT ) as msg: - for element in msg.values: - values.append(_deserialize(combine_bytes(element), from_bytes=True)) + values = (_deserialize(combine_bytes(element), from_bytes=True) for element in msg.values) return iterable_type(values) From 6fe6d8bcd3201874b6704db7e3e8ba2a92da1b64 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:32:54 +0200 Subject: [PATCH 13/83] Update recursive_primitives.py --- packages/syft/src/syft/serde/recursive_primitives.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/serde/recursive_primitives.py b/packages/syft/src/syft/serde/recursive_primitives.py index a536950b5b7..5bc2720a353 100644 --- a/packages/syft/src/syft/serde/recursive_primitives.py +++ b/packages/syft/src/syft/serde/recursive_primitives.py @@ -64,6 +64,7 @@ def serialize_iterable(iterable: Collection) -> bytes: def deserialize_iterable(iterable_type: type, blob: bytes) -> Collection: + # relative from .deserialize import _deserialize MAX_TRAVERSAL_LIMIT = 2**64 - 1 @@ -71,7 +72,10 @@ def deserialize_iterable(iterable_type: type, blob: bytes) -> Collection: with iterable_schema.from_bytes( blob, traversal_limit_in_words=MAX_TRAVERSAL_LIMIT ) as msg: - values = (_deserialize(combine_bytes(element), from_bytes=True) for element in msg.values) + values = ( + _deserialize(combine_bytes(element), from_bytes=True) + for element in msg.values + ) return iterable_type(values) From ba697c34aa8eb4065a3f179575b0285c68db2b5b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:36:29 +0200 Subject: [PATCH 14/83] Update action_object.py --- packages/syft/src/syft/service/action/action_object.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index dffa3d3d9de..a0377f3d229 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1090,11 +1090,9 @@ def syft_make_action( if kwargs is None: kwargs = {} - arg_ids = [] - kwarg_ids = {} + arg_ids = [self._syft_prepare_obj_uid(obj) for obj in args] - for obj in args: - arg_ids.append(self._syft_prepare_obj_uid(obj)) + kwarg_ids = {} for k, obj in kwargs.items(): kwarg_ids[k] = self._syft_prepare_obj_uid(obj) From dab58914067f4712fc4f991b89d312e257f92998 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:38:16 +0200 Subject: [PATCH 15/83] Update action_object.py --- packages/syft/src/syft/service/action/action_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index a0377f3d229..15ef93096c5 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -2170,7 +2170,7 @@ def has_action_data_empty(args: Any, kwargs: Any) -> bool: if is_action_data_empty(a): return True - for _, a in kwargs.items(): + for a in kwargs.values(): if is_action_data_empty(a): return True return False From 0bb036c25382aba4c1f8a39111fd2a38e8f3e230 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:41:51 +0200 Subject: [PATCH 16/83] Update api_service.py --- packages/syft/src/syft/service/api/api_service.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/syft/src/syft/service/api/api_service.py b/packages/syft/src/syft/service/api/api_service.py index 55d2df05bf0..5ac2fa02dad 100644 --- a/packages/syft/src/syft/service/api/api_service.py +++ b/packages/syft/src/syft/service/api/api_service.py @@ -238,11 +238,7 @@ def api_endpoints( return SyftError(message=result.err()) all_api_endpoints = result.ok() - api_endpoint_view = [] - for api_endpoint in all_api_endpoints: - api_endpoint_view.append( - api_endpoint.to(TwinAPIEndpointView, context=context) - ) + api_endpoint_view = [api_endpoint.to(TwinAPIEndpointView, context=context) for api_endpoint in all_api_endpoints] return api_endpoint_view From e6457801517931f46f15ec30e49c365e315f2d4e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:43:06 +0200 Subject: [PATCH 17/83] Update api_service.py --- packages/syft/src/syft/service/api/api_service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/api/api_service.py b/packages/syft/src/syft/service/api/api_service.py index 5ac2fa02dad..6c6b4da2cde 100644 --- a/packages/syft/src/syft/service/api/api_service.py +++ b/packages/syft/src/syft/service/api/api_service.py @@ -238,7 +238,10 @@ def api_endpoints( return SyftError(message=result.err()) all_api_endpoints = result.ok() - api_endpoint_view = [api_endpoint.to(TwinAPIEndpointView, context=context) for api_endpoint in all_api_endpoints] + api_endpoint_view = [ + api_endpoint.to(TwinAPIEndpointView, context=context) + for api_endpoint in all_api_endpoints + ] return api_endpoint_view From f08f428a0bb6034b4446736fdd2f631d7e157b9c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:44:13 +0200 Subject: [PATCH 18/83] Update api_service.py --- packages/syft/src/syft/service/api/api_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/api/api_service.py b/packages/syft/src/syft/service/api/api_service.py index 6c6b4da2cde..4084814001c 100644 --- a/packages/syft/src/syft/service/api/api_service.py +++ b/packages/syft/src/syft/service/api/api_service.py @@ -239,7 +239,7 @@ def api_endpoints( all_api_endpoints = result.ok() api_endpoint_view = [ - api_endpoint.to(TwinAPIEndpointView, context=context) + api_endpoint.to(TwinAPIEndpointView, context=context) for api_endpoint in all_api_endpoints ] From 2c692e163faf9c1392f9bd1d028aa59b6a2bdedc Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:55:02 +0200 Subject: [PATCH 19/83] Update user_code.py --- packages/syft/src/syft/service/code/user_code.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 2cbeaf31967..832379fb869 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -442,7 +442,7 @@ def _get_input_policy(self) -> InputPolicy | None: ): # TODO: Tech Debt here node_view_workaround = False - for k, _ in self.input_policy_init_kwargs.items(): + for k in self.input_policy_init_kwargs.keys(): if isinstance(k, NodeIdentity): node_view_workaround = True @@ -727,7 +727,7 @@ def _inner_repr(self, level: int = 0) -> str: [f"{' '*level}{substring}" for substring in md.split("\n")[:-1]] ) if self.nested_codes is not None: - for _, (obj, _) in self.nested_codes.items(): + for obj, _ in self.nested_codes.values(): code = obj.resolve md += "\n" md += code._inner_repr(level=level + 1) @@ -876,7 +876,7 @@ def _ephemeral_node_call( # And need only ActionObjects # Also, this works only on the assumption that all inputs # are ActionObjects, which might change in the future - for _, id in obj_dict.items(): + for id in obj_dict.values(): mock_obj = api.services.action.get_mock(id) if isinstance(mock_obj, SyftError): data_obj = api.services.action.get(id) From 32b43516453753131da5497cfe71c87adfebb663 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 09:58:07 +0200 Subject: [PATCH 20/83] Update code_history.py --- packages/syft/src/syft/service/code_history/code_history.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/code_history/code_history.py b/packages/syft/src/syft/service/code_history/code_history.py index 139fd9e2d7a..34dc7174aa2 100644 --- a/packages/syft/src/syft/service/code_history/code_history.py +++ b/packages/syft/src/syft/service/code_history/code_history.py @@ -137,7 +137,5 @@ def __getitem__(self, key: str | int) -> CodeHistoriesDict | SyftError: return api.services.code_history.get_history_for_user(key) def _repr_html_(self) -> str: - rows = [] - for user, funcs in self.user_dict.items(): - rows += [{"user": user, "UserCodes": funcs}] + rows = [{"user": user, "UserCodes": funcs} for user, funcs in self.user_dict.items()] return create_table_template(rows, "UserCodeHistory", icon=None) From 011b46903ae99472fae575d0681307b6b9b6e6b6 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 10:00:35 +0200 Subject: [PATCH 21/83] Update dataset.py --- packages/syft/src/syft/service/dataset/dataset.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 687a2eb5f84..8bbcf83cf2d 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -484,10 +484,7 @@ def _repr_html_(self) -> Any: """ def action_ids(self) -> list[UID]: - data = [] - for asset in self.asset_list: - if asset.action_id: - data.append(asset.action_id) + data = [asset.action_id for asset in self.asset_list if asset.action_id] return data @property From d9cf2da64a44060632caec3b8dd24195c4ca378d Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 10:03:43 +0200 Subject: [PATCH 22/83] Update dataset_service.py --- .../syft/service/dataset/dataset_service.py | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/packages/syft/src/syft/service/dataset/dataset_service.py b/packages/syft/src/syft/service/dataset/dataset_service.py index f791d6b9dc2..7e9e6b30758 100644 --- a/packages/syft/src/syft/service/dataset/dataset_service.py +++ b/packages/syft/src/syft/service/dataset/dataset_service.py @@ -177,26 +177,19 @@ def get_by_action_id( return datasets return SyftError(message=result.err()) - @service_method( - path="dataset.get_assets_by_action_id", - name="get_assets_by_action_id", - roles=DATA_SCIENTIST_ROLE_LEVEL, - ) - def get_assets_by_action_id( - self, context: AuthedServiceContext, uid: UID - ) -> list[Asset] | SyftError: - """Get Assets by an Action ID""" - datasets = self.get_by_action_id(context=context, uid=uid) - assets = [] - if isinstance(datasets, list): - for dataset in datasets: - for asset in dataset.asset_list: - if asset.action_id == uid: - assets.append(asset) - return assets - elif isinstance(datasets, SyftError): - return datasets - return [] +@service_method( + path="dataset.get_assets_by_action_id", + name="get_assets_by_action_id", + roles=DATA_SCIENTIST_ROLE_LEVEL, +) +def get_assets_by_action_id( + self, context: AuthedServiceContext, uid: UID +) -> list[Asset] | SyftError: + """Get Assets by an Action ID""" + datasets = self.get_by_action_id(context=context, uid=uid) + if isinstance(datasets, SyftError): + return datasets + return [asset for dataset in datasets for asset in dataset.asset_list if asset.action_id == uid] @service_method( path="dataset.delete_by_uid", From dc81317fd93c45e870f08f7fa462f9531246b6b5 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 10:07:07 +0200 Subject: [PATCH 23/83] Update dataset_service.py --- packages/syft/src/syft/service/dataset/dataset_service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/dataset/dataset_service.py b/packages/syft/src/syft/service/dataset/dataset_service.py index 7e9e6b30758..3a731d7fe9f 100644 --- a/packages/syft/src/syft/service/dataset/dataset_service.py +++ b/packages/syft/src/syft/service/dataset/dataset_service.py @@ -177,6 +177,7 @@ def get_by_action_id( return datasets return SyftError(message=result.err()) + @service_method( path="dataset.get_assets_by_action_id", name="get_assets_by_action_id", @@ -189,7 +190,12 @@ def get_assets_by_action_id( datasets = self.get_by_action_id(context=context, uid=uid) if isinstance(datasets, SyftError): return datasets - return [asset for dataset in datasets for asset in dataset.asset_list if asset.action_id == uid] + return [ + asset + for dataset in datasets + for asset in dataset.asset_list + if asset.action_id == uid + ] @service_method( path="dataset.delete_by_uid", From 728addc01ccdfe18c8b3687d942d45b795d847af Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 10:07:53 +0200 Subject: [PATCH 24/83] Update code_history.py --- packages/syft/src/syft/service/code_history/code_history.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/code_history/code_history.py b/packages/syft/src/syft/service/code_history/code_history.py index 34dc7174aa2..b5e893c87bf 100644 --- a/packages/syft/src/syft/service/code_history/code_history.py +++ b/packages/syft/src/syft/service/code_history/code_history.py @@ -137,5 +137,7 @@ def __getitem__(self, key: str | int) -> CodeHistoriesDict | SyftError: return api.services.code_history.get_history_for_user(key) def _repr_html_(self) -> str: - rows = [{"user": user, "UserCodes": funcs} for user, funcs in self.user_dict.items()] + rows = [ + {"user": user, "UserCodes": funcs} for user, funcs in self.user_dict.items() + ] return create_table_template(rows, "UserCodeHistory", icon=None) From d2e85861106bc388005e7ca03f181c1e8c4d69aa Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 10:13:07 +0200 Subject: [PATCH 25/83] Update dataset_service.py --- .../syft/service/dataset/dataset_service.py | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/packages/syft/src/syft/service/dataset/dataset_service.py b/packages/syft/src/syft/service/dataset/dataset_service.py index 3a731d7fe9f..451746fa15a 100644 --- a/packages/syft/src/syft/service/dataset/dataset_service.py +++ b/packages/syft/src/syft/service/dataset/dataset_service.py @@ -177,25 +177,24 @@ def get_by_action_id( return datasets return SyftError(message=result.err()) - -@service_method( - path="dataset.get_assets_by_action_id", - name="get_assets_by_action_id", - roles=DATA_SCIENTIST_ROLE_LEVEL, -) -def get_assets_by_action_id( - self, context: AuthedServiceContext, uid: UID -) -> list[Asset] | SyftError: - """Get Assets by an Action ID""" - datasets = self.get_by_action_id(context=context, uid=uid) - if isinstance(datasets, SyftError): - return datasets - return [ - asset - for dataset in datasets - for asset in dataset.asset_list - if asset.action_id == uid - ] + @service_method( + path="dataset.get_assets_by_action_id", + name="get_assets_by_action_id", + roles=DATA_SCIENTIST_ROLE_LEVEL, + ) + def get_assets_by_action_id( + self, context: AuthedServiceContext, uid: UID + ) -> list[Asset] | SyftError: + """Get Assets by an Action ID""" + datasets = self.get_by_action_id(context=context, uid=uid) + if isinstance(datasets, SyftError): + return datasets + return [ + asset + for dataset in datasets + for asset in dataset.asset_list + if asset.action_id == uid + ] @service_method( path="dataset.delete_by_uid", From c27cbc020ff3b446fd6af5295232054440f1edee Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 10:20:13 +0200 Subject: [PATCH 26/83] Update network_service.py --- .../syft/src/syft/service/network/network_service.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index b38d822c7f4..874baf28e30 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -900,14 +900,10 @@ def _get_association_requests_by_peer_id( RequestService.get_all ) all_requests: list[Request] = request_get_all_method(context) - association_requests: list[Request] = [] - for request in all_requests: - for change in request.changes: - if ( - isinstance(change, AssociationRequestChange) - and change.remote_peer.id == peer_id - ): - association_requests.append(request) + association_requests: list[Request] = [ + request for request in all_requests + if any(isinstance(change, AssociationRequestChange) and change.remote_peer.id == peer_id for change in request.changes) + ] return sorted( association_requests, key=lambda request: request.request_time.utc_timestamp From 1d36ede91f993228a5963578f9591d4e4c70016f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 10:21:54 +0200 Subject: [PATCH 27/83] Update network_service.py --- .../syft/src/syft/service/network/network_service.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 874baf28e30..ac329420168 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -901,8 +901,13 @@ def _get_association_requests_by_peer_id( ) all_requests: list[Request] = request_get_all_method(context) association_requests: list[Request] = [ - request for request in all_requests - if any(isinstance(change, AssociationRequestChange) and change.remote_peer.id == peer_id for change in request.changes) + request + for request in all_requests + if any( + isinstance(change, AssociationRequestChange) + and change.remote_peer.id == peer_id + for change in request.changes + ) ] return sorted( From d85d743c65ef499296058ae6f3c11f20159bf86e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 15:55:48 +0200 Subject: [PATCH 28/83] Update policy.py --- .../syft/src/syft/service/policy/policy.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index 5bf5739d6fc..736bf85407c 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -72,9 +72,7 @@ def filter_only_uids(results: Any) -> list[UID] | dict[str, UID] | UID: results = [results] if isinstance(results, list): - output_list = [] - for v in results: - output_list.append(extract_uid(v)) + output_list = [extract_uid(v) for v in results] return output_list elif isinstance(results, dict): output_dict = {} @@ -712,14 +710,16 @@ def process_class_code(raw_code: str, class_name: str) -> str: "Tuple", "Type", ] - for typing_type in typing_types: - new_body.append( - ast.ImportFrom( - module="typing", - names=[ast.alias(name=typing_type, asname=typing_type)], - level=0, - ) + new_body.append( + ast.ImportFrom( + module="typing", + names=[ + ast.alias(name=typing_type, asname=typing_type) + for typing_type in typing_types + ], + level=0, ) + ) new_body.append(new_class) module = ast.Module(new_body, type_ignores=[]) try: From 69ef9ed92df10d476a2c69d1e785091158ed4718 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 15:59:15 +0200 Subject: [PATCH 29/83] Update project.py --- packages/syft/src/syft/service/project/project.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index 981f7ff9192..3af678746de 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -939,12 +939,11 @@ def create_code_request( reason=reason, ) - def get_messages(self) -> list[ProjectMessage | ProjectThreadMessage]: - messages = [] - for event in self.events: - if isinstance(event, ProjectMessage | ProjectThreadMessage): - messages.append(event) - return messages + def get_messages(self) -> list[ProjectMessage, ProjectThreadMessage]: + return [event + for event in self.events + if isinstance(event, (ProjectMessage, ProjectThreadMessage)) + ] @property def messages(self) -> str: From 50717ee46bd499650821a2fbc6ccc5ae11bca71b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:01:16 +0200 Subject: [PATCH 30/83] Update project.py --- packages/syft/src/syft/service/project/project.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index 3af678746de..d76a45c4250 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -940,10 +940,11 @@ def create_code_request( ) def get_messages(self) -> list[ProjectMessage, ProjectThreadMessage]: - return [event - for event in self.events - if isinstance(event, (ProjectMessage, ProjectThreadMessage)) - ] + return [ + event + for event in self.events + if isinstance(event, (ProjectMessage, ProjectThreadMessage)) + ] @property def messages(self) -> str: From fd4800e6d158d9f725a72aba9113de3a677ec8f4 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:03:07 +0200 Subject: [PATCH 31/83] Update project.py --- packages/syft/src/syft/service/project/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index d76a45c4250..3bdbc086d39 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -942,7 +942,7 @@ def create_code_request( def get_messages(self) -> list[ProjectMessage, ProjectThreadMessage]: return [ event - for event in self.events + for event in self.events if isinstance(event, (ProjectMessage, ProjectThreadMessage)) ] From e4c20397346e6b0baf9db1694c923501d44f60cb Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:06:47 +0200 Subject: [PATCH 32/83] Update project.py --- packages/syft/src/syft/service/project/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index 3bdbc086d39..3eba53ab9a1 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -939,7 +939,7 @@ def create_code_request( reason=reason, ) - def get_messages(self) -> list[ProjectMessage, ProjectThreadMessage]: + def get_messages(self) -> list[ProjectMessage | ProjectThreadMessage]: return [ event for event in self.events From ba3f0ad383ef55dc705daaec23b929f841f19278 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:25:41 +0200 Subject: [PATCH 33/83] Update zmq_queue.py --- .../syft/src/syft/service/queue/zmq_queue.py | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 3ad4b732f89..ea53a50b73a 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -300,8 +300,7 @@ def read_items(self) -> None: continue for arg in action.args: self.preprocess_action_arg(arg) - for _, arg in action.kwargs.items(): - self.preprocess_action_arg(arg) + [self.preprocess_action_arg(arg) for _, arg in action.kwargs.items()] msg_bytes = serialize(item, to_bytes=True) worker_pool = item.worker_pool.resolve_with_context( @@ -476,8 +475,38 @@ def _run(self) -> None: if self._stop.is_set(): return - for _, service in self.services.items(): - self.dispatch(service, None) + def _run(self) -> None: + while True: + if self._stop.is_set(): + return + + for service in self.services.keys(): + thread = threading.Thread(target=self.dispatch, args=(service, None)) + thread.start() + + items = None + + try: + items = self.poll_workers.poll(ZMQ_POLLER_TIMEOUT_MSEC) + except Exception as e: + logger.exception("Failed to poll items: {}", e) + + if items: + msg = self.socket.recv_multipart() + + logger.debug("Recieve: {}", msg) + + address = msg.pop(0) + empty = msg.pop(0) # noqa: F841 + header = msg.pop(0) + + if header == QueueMsgProtocol.W_WORKER: + self.process_worker(address, msg) + else: + logger.error("Invalid message header: {}", header) + + self.send_heartbeats() + self.purge_workers() items = None From 7dd349e36dc20878543492a274d17022e8e1194f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:28:43 +0200 Subject: [PATCH 34/83] Update zmq_queue.py --- packages/syft/src/syft/service/queue/zmq_queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index ea53a50b73a..f1e3fbd8f84 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -957,12 +957,12 @@ def send_message( def close(self) -> SyftError | SyftSuccess: try: - for _, consumers in self.consumers.items(): + for consumers in self.consumers.values(): for consumer in consumers: # make sure look is stopped consumer.close() - for _, producer in self.producers.items(): + for producer in self.producers.values(): # make sure loop is stopped producer.close() # close existing connection. From 2cde63933aee3c8b0d0001e0da14d4e2f9417e64 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:36:15 +0200 Subject: [PATCH 35/83] Update request.py --- packages/syft/src/syft/service/request/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 2a8ae61bf60..0a09085d90f 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -1250,7 +1250,7 @@ def code(self) -> UserCode: def codes(self) -> list[UserCode]: def recursive_code(node: Any) -> list: codes = [] - for _, (obj, new_node) in node.items(): + for (obj, new_node) in node.values(): codes.append(obj.resolve) codes.extend(recursive_code(new_node)) return codes From d0e3e64cedfa8d2074ee54fc3644dc5c10b02573 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:38:08 +0200 Subject: [PATCH 36/83] Update user_roles.py --- packages/syft/src/syft/service/user/user_roles.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/user/user_roles.py b/packages/syft/src/syft/service/user/user_roles.py index 6ed7f4a9796..34ab6d1ede6 100644 --- a/packages/syft/src/syft/service/user/user_roles.py +++ b/packages/syft/src/syft/service/user/user_roles.py @@ -34,9 +34,7 @@ class ServiceRole(Enum): # @property @classmethod def roles_descending(cls) -> list[tuple[int, Self]]: - tuples = [] - for x in cls: - tuples.append((x.value, x)) + tuples = [(x.value, x) for x in cls] return sorted(tuples, reverse=True) @classmethod From d064c9cb02f7b0ff208c2cd35b2d81ea2959a92e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:41:20 +0200 Subject: [PATCH 37/83] Update worker_pool.py --- .../syft/src/syft/service/worker/worker_pool.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/service/worker/worker_pool.py b/packages/syft/src/syft/service/worker/worker_pool.py index 24325ed2995..239ff6fde0a 100644 --- a/packages/syft/src/syft/service/worker/worker_pool.py +++ b/packages/syft/src/syft/service/worker/worker_pool.py @@ -180,10 +180,11 @@ def image(self) -> SyftWorkerImage | SyftError | None: @property def running_workers(self) -> list[SyftWorker] | SyftError: """Query the running workers using an API call to the server""" - _running_workers = [] - for worker in self.workers: - if worker.status == WorkerStatus.RUNNING: - _running_workers.append(worker) + _running_workers = [ + worker + for worker in self.workers + if worker.status == WorkerStatus.RUNNING + ] return _running_workers @@ -194,9 +195,11 @@ def healthy_workers(self) -> list[SyftWorker] | SyftError: """ _healthy_workers = [] - for worker in self.workers: - if worker.healthcheck == WorkerHealth.HEALTHY: - _healthy_workers.append(worker) + _healthy_workers = [ + worker + for worker in self.workers + if worker.healthcheck == WorkerHealth.HEALTHY + ] return _healthy_workers From f385c9377b8962349ed10b2aced7af91391be047 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:42:52 +0200 Subject: [PATCH 38/83] Update request.py --- packages/syft/src/syft/service/request/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 0a09085d90f..1a5ab9ff895 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -1250,7 +1250,7 @@ def code(self) -> UserCode: def codes(self) -> list[UserCode]: def recursive_code(node: Any) -> list: codes = [] - for (obj, new_node) in node.values(): + for obj, new_node in node.values(): codes.append(obj.resolve) codes.extend(recursive_code(new_node)) return codes From 452124cc3f14bdce72b882be80cc9c6100bce9a0 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:44:38 +0200 Subject: [PATCH 39/83] Update worker_pool.py --- packages/syft/src/syft/service/worker/worker_pool.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/worker/worker_pool.py b/packages/syft/src/syft/service/worker/worker_pool.py index 239ff6fde0a..b3ad441d0f6 100644 --- a/packages/syft/src/syft/service/worker/worker_pool.py +++ b/packages/syft/src/syft/service/worker/worker_pool.py @@ -181,9 +181,7 @@ def image(self) -> SyftWorkerImage | SyftError | None: def running_workers(self) -> list[SyftWorker] | SyftError: """Query the running workers using an API call to the server""" _running_workers = [ - worker - for worker in self.workers - if worker.status == WorkerStatus.RUNNING + worker for worker in self.workers if worker.status == WorkerStatus.RUNNING ] return _running_workers From 502a53677890225ae2d04ce02fe601119a700046 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:46:03 +0200 Subject: [PATCH 40/83] Update project.py --- packages/syft/src/syft/service/project/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index 3eba53ab9a1..e768154ed00 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -943,7 +943,7 @@ def get_messages(self) -> list[ProjectMessage | ProjectThreadMessage]: return [ event for event in self.events - if isinstance(event, (ProjectMessage, ProjectThreadMessage)) + if isinstance(event, (ProjectMessage | ProjectThreadMessage)) ] @property From 501c472467bfd8d629bc4fee88f484a859a5b8ca Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:47:49 +0200 Subject: [PATCH 41/83] Update zmq_queue.py --- packages/syft/src/syft/service/queue/zmq_queue.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index f1e3fbd8f84..e60a7d5a942 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -475,11 +475,6 @@ def _run(self) -> None: if self._stop.is_set(): return - def _run(self) -> None: - while True: - if self._stop.is_set(): - return - for service in self.services.keys(): thread = threading.Thread(target=self.dispatch, args=(service, None)) thread.start() From 498eb2b58cae53796f4a241b1691ac6d97afe777 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:49:45 +0200 Subject: [PATCH 42/83] Update zmq_queue.py --- .../syft/src/syft/service/queue/zmq_queue.py | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index e60a7d5a942..6b6f7cdda75 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -503,30 +503,6 @@ def _run(self) -> None: self.send_heartbeats() self.purge_workers() - items = None - - try: - items = self.poll_workers.poll(ZMQ_POLLER_TIMEOUT_MSEC) - except Exception as e: - logger.exception("Failed to poll items: {}", e) - - if items: - msg = self.socket.recv_multipart() - - logger.debug("Recieve: {}", msg) - - address = msg.pop(0) - empty = msg.pop(0) # noqa: F841 - header = msg.pop(0) - - if header == QueueMsgProtocol.W_WORKER: - self.process_worker(address, msg) - else: - logger.error("Invalid message header: {}", header) - - self.send_heartbeats() - self.purge_workers() - def require_worker(self, address: bytes) -> Worker: """Finds the worker (creates if necessary).""" identity = hexlify(address) From 2049f5260c2aa4cddc9a4e17aba1e01b2e39ceda Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:52:48 +0200 Subject: [PATCH 43/83] Update worker_pool_service.py --- packages/syft/src/syft/service/worker/worker_pool_service.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 9e7d02572c1..998eefb8462 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -362,9 +362,7 @@ def get_all( return SyftError(message=f"{result.err()}") worker_pools: list[WorkerPool] = result.ok() - res: list[tuple] = [] - for pool in worker_pools: - res.append((pool.name, pool)) + res = [(pool.name, pool) for pool in worker_pools] return DictTuple(res) @service_method( From 427e67a7f3a0b7414cabf9b51eabbd8c0cab5c02 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:56:02 +0200 Subject: [PATCH 44/83] Update seaweedfs.py --- .../src/syft/store/blob_storage/seaweedfs.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index 1d88fedda37..930c99ce8a7 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -115,23 +115,26 @@ def async_generator( def add_chunks_to_queue( self, + data: IO[bytes], + part_size: int, queue: Queue, chunk_size: int = DEFAULT_UPLOAD_CHUNK_SIZE, ) -> None: - """Creates a data geneator for the part""" + """Creates a data generator for the part""" n = 0 - - while n * chunk_size <= part_size: - try: - chunk = data.read(chunk_size) - self.no_lines += chunk.count(b"\n") - n += 1 - queue.put(chunk) - except BlockingIOError: - # if end of file, stop - queue.put(0) - # if end of part, stop - queue.put(0) + + while True: + if n * chunk_size >= part_size: + break + chunk = data.read(chunk_size) + if not chunk: + break + self.no_lines += chunk.count(b"\n") + n += 1 + queue.put(chunk) + + # Use None to indicate the end of the part or file + queue.put(None) gen = PartGenerator() From 1c7b030b78333124f76c1318d5e4c4937f76f213 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 16:59:51 +0200 Subject: [PATCH 45/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index 7b0f2330875..df736a28b70 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -139,11 +139,12 @@ def syft_iter_content( except requests.exceptions.RequestException as e: if attempt < max_retries: - print( + logging.error( f"Attempt {attempt}/{max_retries} failed: {e} at byte {current_byte}. Retrying..." ) + time.sleep(2 ** attempt) # exponential backoff else: - print(f"Max retries reached. Failed with error: {e}") + logging.error(f"Max retries reached. Failed with error: {e}") raise From 3b7dbaa628a1aff56f83d32192306597cf5f9881 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:01:55 +0200 Subject: [PATCH 46/83] Update dict_document_store.py --- packages/syft/src/syft/store/dict_document_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/store/dict_document_store.py b/packages/syft/src/syft/store/dict_document_store.py index d0f6d9cb51f..71f0eced1f1 100644 --- a/packages/syft/src/syft/store/dict_document_store.py +++ b/packages/syft/src/syft/store/dict_document_store.py @@ -80,7 +80,7 @@ def __init__( ) def reset(self) -> None: - for _, partition in self.partitions.items(): + for partition in self.partitions.values(): partition.prune() From 46cec0c32f5f7924e85cadfa5c88a92b35478d9c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:05:10 +0200 Subject: [PATCH 47/83] Update mongo_document_store.py --- packages/syft/src/syft/store/mongo_document_store.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/store/mongo_document_store.py b/packages/syft/src/syft/store/mongo_document_store.py index 59d6799c2bb..68c00957bd9 100644 --- a/packages/syft/src/syft/store/mongo_document_store.py +++ b/packages/syft/src/syft/store/mongo_document_store.py @@ -422,12 +422,11 @@ def _get_all_from_store( syft_objs.append(obj.to(self.settings.object_type, transform_context)) # TODO: maybe do this in loop before this - res = [] - for s in syft_objs: - if has_permission or self.has_permission( - ActionObjectREAD(uid=s.id, credentials=credentials) - ): - res.append(s) + res = [ + s + for s in syft_objs + if has_permission or self.has_permission(ActionObjectREAD(uid=s.id, credentials=credentials)) + ] return Ok(res) def _delete( From 786d9dd04fcd85cf110abb60afffc6c1e2bd632d Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:07:35 +0200 Subject: [PATCH 48/83] Update sqlite_document_store.py --- packages/syft/src/syft/store/sqlite_document_store.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py index 8ef1b2803a8..96a0b70b81f 100644 --- a/packages/syft/src/syft/store/sqlite_document_store.py +++ b/packages/syft/src/syft/store/sqlite_document_store.py @@ -266,7 +266,6 @@ def _get_all(self) -> Any: def _get_all_keys(self) -> Any: select_sql = f"select uid from {self.table_name} order by sqltime" # nosec - keys = [] res = self._execute(select_sql) if res.is_err(): @@ -277,8 +276,7 @@ def _get_all_keys(self) -> Any: if rows is None: return [] - for row in rows: - keys.append(UID(row[0])) + keys = [UID(row[0]) for row in rows] return keys def _delete(self, key: UID) -> None: From d03914fda58a0cf19fa50da5b876d41ceb6e3c88 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:08:08 +0200 Subject: [PATCH 49/83] Update schema.py --- packages/syft/src/syft/util/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/util/schema.py b/packages/syft/src/syft/util/schema.py index f918ed0d4af..cc0958eddda 100644 --- a/packages/syft/src/syft/util/schema.py +++ b/packages/syft/src/syft/util/schema.py @@ -179,7 +179,7 @@ def process_type_bank(type_bank: dict[str, tuple[Any, ...]]) -> dict[str, dict]: def resolve_references(json_mappings: dict[str, dict]) -> dict[str, dict]: # track second pass generated types new_types = {} - for _, json_schema in json_mappings.items(): + for json_schema in json_mappings.values(): replace_types = {} for attribute, config in json_schema["properties"].items(): if "type" in config: From dd46b16238caa133907e6a9a329a1bfe94b5472b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:10:40 +0200 Subject: [PATCH 50/83] Update dicttuple_test.py --- packages/syft/tests/syft/types/dicttuple_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/tests/syft/types/dicttuple_test.py b/packages/syft/tests/syft/types/dicttuple_test.py index de32f2545bc..1a0405b406f 100644 --- a/packages/syft/tests/syft/types/dicttuple_test.py +++ b/packages/syft/tests/syft/types/dicttuple_test.py @@ -39,9 +39,7 @@ def test_dict_tuple_not_subclassing_mapping(): @pytest.mark.parametrize("dict_tuple", SIMPLE_TEST_CASES) def test_should_iter_over_value(dict_tuple: DictTuple) -> None: - values = [] - for v in dict_tuple: - values.append(v) + values = [v for v in dict_tuple] assert values == [1, 2] From f3f80234ffdba97275da60e3d09bfb3f4076ad36 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:11:36 +0200 Subject: [PATCH 51/83] Update user_service_test.py --- packages/syft/tests/syft/users/user_service_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/tests/syft/users/user_service_test.py b/packages/syft/tests/syft/users/user_service_test.py index 7c0cc32562a..cd6d0aeefd4 100644 --- a/packages/syft/tests/syft/users/user_service_test.py +++ b/packages/syft/tests/syft/users/user_service_test.py @@ -219,7 +219,7 @@ def test_userservice_search( guest_user: User, ) -> None: def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Ok | Err: - for key, _ in kwargs.items(): + for key in kwargs.keys(): if hasattr(guest_user, key): return Ok([guest_user]) return Err("Invalid kwargs") From 7566fc391738463d7814a5ce8bd518a3a7941625 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:15:01 +0200 Subject: [PATCH 52/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index df736a28b70..07d2be0587d 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -142,7 +142,7 @@ def syft_iter_content( logging.error( f"Attempt {attempt}/{max_retries} failed: {e} at byte {current_byte}. Retrying..." ) - time.sleep(2 ** attempt) # exponential backoff + time.sleep(2**attempt) # exponential backoff else: logging.error(f"Max retries reached. Failed with error: {e}") raise From a046c54b948bd7ccf8904edae13c127d132a2257 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:15:37 +0200 Subject: [PATCH 53/83] Update seaweedfs.py --- packages/syft/src/syft/store/blob_storage/seaweedfs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index 930c99ce8a7..1f94e94cefb 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -122,7 +122,7 @@ def add_chunks_to_queue( ) -> None: """Creates a data generator for the part""" n = 0 - + while True: if n * chunk_size >= part_size: break @@ -132,7 +132,7 @@ def add_chunks_to_queue( self.no_lines += chunk.count(b"\n") n += 1 queue.put(chunk) - + # Use None to indicate the end of the part or file queue.put(None) From d5c013d27b537a99f3104f443d6265630feaa5e7 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:16:12 +0200 Subject: [PATCH 54/83] Update mongo_document_store.py --- packages/syft/src/syft/store/mongo_document_store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/store/mongo_document_store.py b/packages/syft/src/syft/store/mongo_document_store.py index 68c00957bd9..234dd2c723b 100644 --- a/packages/syft/src/syft/store/mongo_document_store.py +++ b/packages/syft/src/syft/store/mongo_document_store.py @@ -425,7 +425,8 @@ def _get_all_from_store( res = [ s for s in syft_objs - if has_permission or self.has_permission(ActionObjectREAD(uid=s.id, credentials=credentials)) + if has_permission + or self.has_permission(ActionObjectREAD(uid=s.id, credentials=credentials)) ] return Ok(res) From 49e82afa2c4a67db1b2e75e2640cc78d0ef2765b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:18:34 +0200 Subject: [PATCH 55/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index 07d2be0587d..08c773f737d 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -139,12 +139,12 @@ def syft_iter_content( except requests.exceptions.RequestException as e: if attempt < max_retries: - logging.error( + print( f"Attempt {attempt}/{max_retries} failed: {e} at byte {current_byte}. Retrying..." ) time.sleep(2**attempt) # exponential backoff else: - logging.error(f"Max retries reached. Failed with error: {e}") + print(f"Max retries reached. Failed with error: {e}") raise From 3be33d3dc8188ba6b37b88b5e157ce53c675cf5a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:21:19 +0200 Subject: [PATCH 56/83] Update zmq_queue.py --- packages/syft/src/syft/service/queue/zmq_queue.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 6b6f7cdda75..2a694642d26 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -300,7 +300,10 @@ def read_items(self) -> None: continue for arg in action.args: self.preprocess_action_arg(arg) - [self.preprocess_action_arg(arg) for _, arg in action.kwargs.items()] + [ + self.preprocess_action_arg(arg) + for arg in action.kwargs.values() + ] msg_bytes = serialize(item, to_bytes=True) worker_pool = item.worker_pool.resolve_with_context( From 21300f5ecf98c23cca557d660e67ca37594ffc0f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:23:07 +0200 Subject: [PATCH 57/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index 08c773f737d..8c99e9d8cb3 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -41,6 +41,7 @@ """ # stdlib +import time from collections.abc import Callable from collections.abc import Generator from io import BytesIO From f3c9736fbfc1d45eb0a55742abc85f9201d8a294 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:29:01 +0200 Subject: [PATCH 58/83] Update seaweedfs.py --- .../src/syft/store/blob_storage/seaweedfs.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index 1f94e94cefb..1d88fedda37 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -115,26 +115,23 @@ def async_generator( def add_chunks_to_queue( self, - data: IO[bytes], - part_size: int, queue: Queue, chunk_size: int = DEFAULT_UPLOAD_CHUNK_SIZE, ) -> None: - """Creates a data generator for the part""" + """Creates a data geneator for the part""" n = 0 - while True: - if n * chunk_size >= part_size: - break - chunk = data.read(chunk_size) - if not chunk: - break - self.no_lines += chunk.count(b"\n") - n += 1 - queue.put(chunk) - - # Use None to indicate the end of the part or file - queue.put(None) + while n * chunk_size <= part_size: + try: + chunk = data.read(chunk_size) + self.no_lines += chunk.count(b"\n") + n += 1 + queue.put(chunk) + except BlockingIOError: + # if end of file, stop + queue.put(0) + # if end of part, stop + queue.put(0) gen = PartGenerator() From 72ab45078925214e54db714c85ffe8963b2c0a6e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:34:59 +0200 Subject: [PATCH 59/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index 8c99e9d8cb3..a0c4cdd5f6b 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -41,10 +41,10 @@ """ # stdlib -import time from collections.abc import Callable from collections.abc import Generator from io import BytesIO +import time from typing import Any # third party From 149369363ad3902d6a93cf6d700a9269c12cde99 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:40:53 +0200 Subject: [PATCH 60/83] Update zmq_queue.py --- packages/syft/src/syft/service/queue/zmq_queue.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 2a694642d26..266728732d7 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -300,10 +300,8 @@ def read_items(self) -> None: continue for arg in action.args: self.preprocess_action_arg(arg) - [ + for arg in action.kwargs.values(): self.preprocess_action_arg(arg) - for arg in action.kwargs.values() - ] msg_bytes = serialize(item, to_bytes=True) worker_pool = item.worker_pool.resolve_with_context( From acf050e9e044218b7946e2368b32bd62bb2e1923 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 17:44:11 +0200 Subject: [PATCH 61/83] Update dicttuple_test.py --- packages/syft/tests/syft/types/dicttuple_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/tests/syft/types/dicttuple_test.py b/packages/syft/tests/syft/types/dicttuple_test.py index 1a0405b406f..71bf1bb52c9 100644 --- a/packages/syft/tests/syft/types/dicttuple_test.py +++ b/packages/syft/tests/syft/types/dicttuple_test.py @@ -39,7 +39,7 @@ def test_dict_tuple_not_subclassing_mapping(): @pytest.mark.parametrize("dict_tuple", SIMPLE_TEST_CASES) def test_should_iter_over_value(dict_tuple: DictTuple) -> None: - values = [v for v in dict_tuple] + values = list(dict_tuple) assert values == [1, 2] From 0d35a03982a43392212b0f7cbf91c910fa8bc481 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:03:39 +0200 Subject: [PATCH 62/83] Update seaweedfs.py --- .../syft/src/syft/store/blob_storage/seaweedfs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index 1d88fedda37..a63ed8a2d67 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -121,16 +121,17 @@ def add_chunks_to_queue( """Creates a data geneator for the part""" n = 0 - while n * chunk_size <= part_size: - try: + try: + while n * chunk_size <= part_size: chunk = data.read(chunk_size) + if not chunk: + break self.no_lines += chunk.count(b"\n") n += 1 queue.put(chunk) - except BlockingIOError: - # if end of file, stop - queue.put(0) - # if end of part, stop + except BlockingIOError: + pass + # if end of file or part, stop queue.put(0) gen = PartGenerator() From 5c733eb7d03e292c7ad593784db6a90e7732bd03 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:10:25 +0200 Subject: [PATCH 63/83] Update ruff.toml --- ruff.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ruff.toml b/ruff.toml index 6d8e8a2f93a..395dfbdef04 100644 --- a/ruff.toml +++ b/ruff.toml @@ -14,6 +14,12 @@ select = [ "F", # pyflake "B", # flake8-bugbear "C4", # flake8-comprehensions + # "NPY", # NumPy-specific rules + # "PD", # pandas-vet + "PERF", # Perflint + # "PL", # Pylint + # "PTH", # flake8-use-pathlib + # "SIM", # flake8-simplify "UP", # pyupgrade ] ignore = [ From 42c1abcc8cc239de8a9cd54e5666ae9040385b86 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:12:50 +0200 Subject: [PATCH 64/83] Update ruff.toml --- ruff.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ruff.toml b/ruff.toml index 395dfbdef04..aa129013555 100644 --- a/ruff.toml +++ b/ruff.toml @@ -14,12 +14,7 @@ select = [ "F", # pyflake "B", # flake8-bugbear "C4", # flake8-comprehensions - # "NPY", # NumPy-specific rules - # "PD", # pandas-vet "PERF", # Perflint - # "PL", # Pylint - # "PTH", # flake8-use-pathlib - # "SIM", # flake8-simplify "UP", # pyupgrade ] ignore = [ From 62ba0767e1e32b19df369b2ba05b9e56d68ee1f6 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:15:39 +0200 Subject: [PATCH 65/83] Update code_history_service.py --- .../src/syft/service/code_history/code_history_service.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/code_history/code_history_service.py b/packages/syft/src/syft/service/code_history/code_history_service.py index f32338ed936..dc356b1f931 100644 --- a/packages/syft/src/syft/service/code_history/code_history_service.py +++ b/packages/syft/src/syft/service/code_history/code_history_service.py @@ -134,9 +134,10 @@ def get_code(uid: UID) -> UserCode | SyftError: code_versions_dict = {} for code_history in code_histories: - user_code_list = [] - for uid in code_history.user_code_history: - user_code_list.append(get_code(uid)) + user_code_list = [ + get_code(uid) + for uid in code_history.user_code_history + ] code_versions = CodeHistoryView( user_code_history=user_code_list, service_func_name=code_history.service_func_name, From 627ec86c9ad61d57c0ff4a39d89714020964f382 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:17:25 +0200 Subject: [PATCH 66/83] Update code_history_service.py --- .../syft/src/syft/service/code_history/code_history_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/code_history/code_history_service.py b/packages/syft/src/syft/service/code_history/code_history_service.py index dc356b1f931..adfd6dbee5d 100644 --- a/packages/syft/src/syft/service/code_history/code_history_service.py +++ b/packages/syft/src/syft/service/code_history/code_history_service.py @@ -135,8 +135,7 @@ def get_code(uid: UID) -> UserCode | SyftError: for code_history in code_histories: user_code_list = [ - get_code(uid) - for uid in code_history.user_code_history + get_code(uid) for uid in code_history.user_code_history ] code_versions = CodeHistoryView( user_code_history=user_code_list, From ad5e84c95044e70d077f6d1a59b3695a52130304 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:39:47 +0200 Subject: [PATCH 67/83] Update __init__.py --- .../syft/src/syft/store/blob_storage/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index a0c4cdd5f6b..6c2a3966219 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -117,16 +117,16 @@ def read(self, _deserialize: bool = True) -> SyftObject | SyftError: def syft_iter_content( - blob_url: str | GridURL, + blob_url: Union[str, GridURL], chunk_size: int, max_retries: int = MAX_RETRIES, timeout: int = DEFAULT_TIMEOUT, ) -> Generator: - """custom iter content with smart retries (start from last byte read)""" + """Custom iter content with smart retries (start from last byte read)""" current_byte = 0 for attempt in range(max_retries): + headers = {"Range": f"bytes={current_byte}-"} try: - headers = {"Range": f"bytes={current_byte}-"} with requests.get( str(blob_url), stream=True, headers=headers, timeout=(timeout, timeout) ) as response: @@ -136,12 +136,11 @@ def syft_iter_content( ): current_byte += len(chunk) yield chunk - return - + return # If successful, exit the function except requests.exceptions.RequestException as e: - if attempt < max_retries: + if attempt < max_retries - 1: print( - f"Attempt {attempt}/{max_retries} failed: {e} at byte {current_byte}. Retrying..." + f"Attempt {attempt + 1}/{max_retries} failed: {e} at byte {current_byte}. Retrying..." ) time.sleep(2**attempt) # exponential backoff else: From 9da3b3a77ac16fd5a5d76b19f111c9e24d59c366 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:42:13 +0200 Subject: [PATCH 68/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index 6c2a3966219..512a984a9f3 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -45,7 +45,7 @@ from collections.abc import Generator from io import BytesIO import time -from typing import Any +from typing import Any, Generator, Union # third party from pydantic import BaseModel From 1332795a8da104c45ed282e276294a29a4c99cd4 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:44:45 +0200 Subject: [PATCH 69/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index 512a984a9f3..2223f883900 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -45,7 +45,8 @@ from collections.abc import Generator from io import BytesIO import time -from typing import Any, Generator, Union +from typing import Any +from typing import Union # third party from pydantic import BaseModel From cd167ca7a203b3f830f799645e9ab1f67e3062fc Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 7 Jun 2024 18:45:33 +0200 Subject: [PATCH 70/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index 2223f883900..9cab0913dde 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -46,7 +46,6 @@ from io import BytesIO import time from typing import Any -from typing import Union # third party from pydantic import BaseModel @@ -118,7 +117,7 @@ def read(self, _deserialize: bool = True) -> SyftObject | SyftError: def syft_iter_content( - blob_url: Union[str, GridURL], + blob_url: str | GridURL, chunk_size: int, max_retries: int = MAX_RETRIES, timeout: int = DEFAULT_TIMEOUT, From ecbdd42c7e719011772fd7fff242af6a1fb3f7aa Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sat, 8 Jun 2024 01:53:01 +0200 Subject: [PATCH 71/83] Update recursive_primitives.py --- packages/syft/src/syft/serde/recursive_primitives.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/serde/recursive_primitives.py b/packages/syft/src/syft/serde/recursive_primitives.py index 5bc2720a353..042234cd843 100644 --- a/packages/syft/src/syft/serde/recursive_primitives.py +++ b/packages/syft/src/syft/serde/recursive_primitives.py @@ -72,10 +72,10 @@ def deserialize_iterable(iterable_type: type, blob: bytes) -> Collection: with iterable_schema.from_bytes( blob, traversal_limit_in_words=MAX_TRAVERSAL_LIMIT ) as msg: - values = ( + values = [ _deserialize(combine_bytes(element), from_bytes=True) for element in msg.values - ) + ] return iterable_type(values) From e4610fe80edba46f9434cf5c786cdb8e31f655fe Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Tue, 11 Jun 2024 06:56:19 +0200 Subject: [PATCH 72/83] Update node.py --- packages/syft/src/syft/node/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 65a3ed80eda..31ead514b2b 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -833,7 +833,6 @@ def get_guest_client(self, verbose: bool = True) -> SyftClient: def __repr__(self) -> str: service_string = "" if not self.is_subprocess: - services = [] services = [service.__name__ for service in self.services] service_string = ", ".join(sorted(services)) service_string = f"\n\nServices:\n{service_string}" From e1196b6e3d3c6ba29190a8ada0dddd0c5d6566c5 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Tue, 11 Jun 2024 06:57:30 +0200 Subject: [PATCH 73/83] Update action_object.py --- packages/syft/src/syft/service/action/action_object.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 15ef93096c5..8bdf940a799 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1092,10 +1092,7 @@ def syft_make_action( arg_ids = [self._syft_prepare_obj_uid(obj) for obj in args] - kwarg_ids = {} - - for k, obj in kwargs.items(): - kwarg_ids[k] = self._syft_prepare_obj_uid(obj) + kwarg_ids = {k: self._syft_prepare_obj_uid(obj) for k, obj in kwargs.items()} action = Action( path=path, From 9f533b49d45df93f72da5a860aa8fc8591d2b380 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Tue, 11 Jun 2024 15:38:17 +0200 Subject: [PATCH 74/83] Update client.py --- packages/syft/src/syft/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 72a048d2d2b..95e31023dd5 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -589,7 +589,7 @@ def get_nested_codes(code: UserCode) -> list[UserCode]: if code.nested_codes is None: return result - for linked_code_obj in code.nested_codes.keys(): + for linked_code_obj in code.nested_codes.values(): nested_code = linked_code_obj.resolve nested_code = deepcopy(nested_code) nested_code.node_uid = code.node_uid From 1d85690ab42bcf1ad8f05dd08fc41264b591a981 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 12 Jun 2024 06:38:48 +0200 Subject: [PATCH 75/83] Update client.py --- packages/syft/src/syft/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 95e31023dd5..dbc8740b421 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -589,7 +589,7 @@ def get_nested_codes(code: UserCode) -> list[UserCode]: if code.nested_codes is None: return result - for linked_code_obj in code.nested_codes.values(): + for (linked_code_obj, _) in code.nested_codes.values(): nested_code = linked_code_obj.resolve nested_code = deepcopy(nested_code) nested_code.node_uid = code.node_uid From 1968fb223dc3ef6936fc16c6d7d1688c0d9432bb Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 12 Jun 2024 06:39:41 +0200 Subject: [PATCH 76/83] Update zmq_queue.py --- packages/syft/src/syft/service/queue/zmq_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 266728732d7..ed8aaa662f2 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -476,7 +476,7 @@ def _run(self) -> None: if self._stop.is_set(): return - for service in self.services.keys(): + for service in self.services.values(): thread = threading.Thread(target=self.dispatch, args=(service, None)) thread.start() From 150771c39578d9b9564bbbdd434d5b9b490a0472 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 12 Jun 2024 06:41:06 +0200 Subject: [PATCH 77/83] Update worker_pool_service.py --- packages/syft/src/syft/service/worker/worker_pool_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 998eefb8462..d42645a19bb 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -362,7 +362,7 @@ def get_all( return SyftError(message=f"{result.err()}") worker_pools: list[WorkerPool] = result.ok() - res = [(pool.name, pool) for pool in worker_pools] + res = ((pool.name, pool) for pool in worker_pools) return DictTuple(res) @service_method( From 8b055fdc5bb6fa434073fd4ce9706b69b1ed67b2 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 12 Jun 2024 06:46:43 +0200 Subject: [PATCH 78/83] Update worker_pool.py --- packages/syft/src/syft/service/worker/worker_pool.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/syft/src/syft/service/worker/worker_pool.py b/packages/syft/src/syft/service/worker/worker_pool.py index b3ad441d0f6..461c4f61b7d 100644 --- a/packages/syft/src/syft/service/worker/worker_pool.py +++ b/packages/syft/src/syft/service/worker/worker_pool.py @@ -191,8 +191,6 @@ def healthy_workers(self) -> list[SyftWorker] | SyftError: """ Query the healthy workers using an API call to the server """ - _healthy_workers = [] - _healthy_workers = [ worker for worker in self.workers From 3ee05ab2f9039b910ada3b65ab99f6bb84e5f767 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 12 Jun 2024 06:52:26 +0200 Subject: [PATCH 79/83] Update __init__.py --- packages/syft/src/syft/store/blob_storage/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index b75c038c0d4..15658ad4c8c 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -44,7 +44,6 @@ from collections.abc import Callable from collections.abc import Generator from io import BytesIO -import time from typing import Any # third party @@ -138,11 +137,10 @@ def syft_iter_content( yield chunk return # If successful, exit the function except requests.exceptions.RequestException as e: - if attempt < max_retries - 1: + if attempt < max_retries: print( - f"Attempt {attempt + 1}/{max_retries} failed: {e} at byte {current_byte}. Retrying..." + f"Attempt {attempt}/{max_retries} failed: {e} at byte {current_byte}. Retrying..." ) - time.sleep(2**attempt) # exponential backoff else: print(f"Max retries reached. Failed with error: {e}") raise From a30d33c84ce19222d34e80e8f7f1adbc7707a481 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 12 Jun 2024 07:24:12 +0200 Subject: [PATCH 80/83] Update zmq_queue.py --- packages/syft/src/syft/service/queue/zmq_queue.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index ed8aaa662f2..6cf4cf3794f 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -477,8 +477,7 @@ def _run(self) -> None: return for service in self.services.values(): - thread = threading.Thread(target=self.dispatch, args=(service, None)) - thread.start() + self.dispatch(service, None) items = None From 736366602eb0f82c13a95c428e6b238ab36ed6f0 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 12 Jun 2024 07:37:47 +0200 Subject: [PATCH 81/83] Update client.py --- packages/syft/src/syft/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index dbc8740b421..f0d9c55b340 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -589,7 +589,7 @@ def get_nested_codes(code: UserCode) -> list[UserCode]: if code.nested_codes is None: return result - for (linked_code_obj, _) in code.nested_codes.values(): + for linked_code_obj, _ in code.nested_codes.values(): nested_code = linked_code_obj.resolve nested_code = deepcopy(nested_code) nested_code.node_uid = code.node_uid From 1149a07dea5f671304d0c233807adde517d5be40 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Wed, 12 Jun 2024 20:17:54 +0800 Subject: [PATCH 82/83] Disable linter rule for the specific syntax test --- packages/syft/tests/syft/types/dicttuple_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/syft/tests/syft/types/dicttuple_test.py b/packages/syft/tests/syft/types/dicttuple_test.py index 71bf1bb52c9..43beb3116c2 100644 --- a/packages/syft/tests/syft/types/dicttuple_test.py +++ b/packages/syft/tests/syft/types/dicttuple_test.py @@ -39,7 +39,9 @@ def test_dict_tuple_not_subclassing_mapping(): @pytest.mark.parametrize("dict_tuple", SIMPLE_TEST_CASES) def test_should_iter_over_value(dict_tuple: DictTuple) -> None: - values = list(dict_tuple) + values = [] + for v in dict_tuple: + values.append(v) # noqa: PERF402 assert values == [1, 2] From 5f918351526cda87770b44c2130294257022393f Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Wed, 12 Jun 2024 20:21:19 +0800 Subject: [PATCH 83/83] Disable ruff PERF rules for now Since most PERF rules' autofixes are unsafe, adding them to run automatically as part of pre-commit would be a bit disruptive. Would be nice to run this from time to time, or enable them once the autofixes are stable. --- ruff.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index aa129013555..3dccdf65b91 100644 --- a/ruff.toml +++ b/ruff.toml @@ -14,7 +14,7 @@ select = [ "F", # pyflake "B", # flake8-bugbear "C4", # flake8-comprehensions - "PERF", # Perflint + # "PERF", # perflint "UP", # pyupgrade ] ignore = [