From 9a01680bb230f585eaaeed1a36158360347d633f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sat, 29 Jun 2024 08:43:01 +0200 Subject: [PATCH 001/123] Fix SIM108 --- packages/syft/src/syft/serde/lib_service_registry.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/syft/src/syft/serde/lib_service_registry.py b/packages/syft/src/syft/serde/lib_service_registry.py index 517df6c643c..66cd5790920 100644 --- a/packages/syft/src/syft/serde/lib_service_registry.py +++ b/packages/syft/src/syft/serde/lib_service_registry.py @@ -243,10 +243,7 @@ def __repr__( ) tree_prefix = "└───" if is_last else "├───" indent_str = "│ " * indent + tree_prefix - if parent_path != "": - path = self.path.replace(f"{parent_path}.", "") - else: - path = self.path + path = self.path.replace(f"{parent_path}.", "") if parent_path != "" else self.path return f"{indent_str}{path} ({self.permissions})\n{children_string}" From 87669d719ad3e8d171380ffa5b79f6d9187f9eba Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sat, 29 Jun 2024 09:00:21 +0200 Subject: [PATCH 002/123] Fix SIM108 --- packages/syft/src/syft/service/queue/zmq_queue.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 08ff386696e..a78ede368f7 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -274,10 +274,7 @@ def contains_nested_actionobjects(self, data: Any) -> bool: def unwrap_collection(col: set | dict | list) -> [Any]: # type: ignore return_values = [] - if isinstance(col, dict): - values = list(col.values()) + list(col.keys()) - else: - values = list(col) + values = list(col.values()) + list(col.keys()) if isinstance(col, dict) else list(col) for v in values: if isinstance(v, list | dict | set): return_values += unwrap_collection(v) From eb87cf73a2a6c8fd7aa36b745afafe58dd02bfb3 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 08:41:22 +0200 Subject: [PATCH 003/123] revert --- packages/syft/src/syft/serde/lib_service_registry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/serde/lib_service_registry.py b/packages/syft/src/syft/serde/lib_service_registry.py index 66cd5790920..517df6c643c 100644 --- a/packages/syft/src/syft/serde/lib_service_registry.py +++ b/packages/syft/src/syft/serde/lib_service_registry.py @@ -243,7 +243,10 @@ def __repr__( ) tree_prefix = "└───" if is_last else "├───" indent_str = "│ " * indent + tree_prefix - path = self.path.replace(f"{parent_path}.", "") if parent_path != "" else self.path + if parent_path != "": + path = self.path.replace(f"{parent_path}.", "") + else: + path = self.path return f"{indent_str}{path} ({self.permissions})\n{children_string}" From 96173ddf768c1876363e9056bd0dc62f709a3aeb Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 08:47:38 +0200 Subject: [PATCH 004/123] revert --- packages/syft/src/syft/service/queue/zmq_queue.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index a78ede368f7..08ff386696e 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -274,7 +274,10 @@ def contains_nested_actionobjects(self, data: Any) -> bool: def unwrap_collection(col: set | dict | list) -> [Any]: # type: ignore return_values = [] - values = list(col.values()) + list(col.keys()) if isinstance(col, dict) else list(col) + if isinstance(col, dict): + values = list(col.values()) + list(col.keys()) + else: + values = list(col) for v in values: if isinstance(v, list | dict | set): return_values += unwrap_collection(v) From 0fcf7b7045f48e79a0cf4d0c421d27b20b395995 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:11:24 +0200 Subject: [PATCH 005/123] Update .ruff to version 0.5.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab1745d6ce5..11f4f4b46e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,7 +84,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.4.7" + rev: "v0.5.0" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] From 686785dcf6c55d9a584b5d8abb6d0a814b4702f6 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:12:09 +0200 Subject: [PATCH 006/123] Update ruff.toml --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index bdf2c46b9cf..355221099de 100644 --- a/ruff.toml +++ b/ruff.toml @@ -15,6 +15,7 @@ select = [ "B", # flake8-bugbear "C4", # flake8-comprehensions # "PERF", # perflint + "PL", # pylint "UP", # pyupgrade ] ignore = [ From b2e1262ff5e96a493a8aefd30924e01e4304ab66 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:19:58 +0200 Subject: [PATCH 007/123] Add ignored codes --- ruff.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ruff.toml b/ruff.toml index 355221099de..c961b4019d7 100644 --- a/ruff.toml +++ b/ruff.toml @@ -19,8 +19,12 @@ select = [ "UP", # pyupgrade ] ignore = [ - "B904", # check for raise statements in exception handlers that lack a from clause - "B905", # zip() without an explicit strict= parameter + "B904", # check for raise statements in exception handlers that lack a from clause + "B905", # zip() without an explicit strict= parameter + "PLR0911", # Too many return statements + "PLR0913", # Too many arguments in function definition + "PLR0915", # Too many statements + ] [lint.per-file-ignores] From 1c3aece9b4d6eae1901ed333647ac0484136d984 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:23:21 +0200 Subject: [PATCH 008/123] Linter --- ruff.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index c961b4019d7..19f5a3a772e 100644 --- a/ruff.toml +++ b/ruff.toml @@ -24,7 +24,6 @@ ignore = [ "PLR0911", # Too many return statements "PLR0913", # Too many arguments in function definition "PLR0915", # Too many statements - ] [lint.per-file-ignores] From 1ddd58b369d66c7dbc5e049ca467fbf8022229e7 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:25:07 +0200 Subject: [PATCH 009/123] Linter --- notebooks/api/0.8/04-pytorch-example.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/api/0.8/04-pytorch-example.ipynb b/notebooks/api/0.8/04-pytorch-example.ipynb index 5cbc89ecaea..341107da536 100644 --- a/notebooks/api/0.8/04-pytorch-example.ipynb +++ b/notebooks/api/0.8/04-pytorch-example.ipynb @@ -25,7 +25,7 @@ "source": [ "# third party\n", "import torch\n", - "import torch.nn as nn\n", + "from torch import nn\n", "import torch.nn.functional as F\n", "\n", "# syft absolute\n", @@ -252,7 +252,7 @@ "def train_mlp(weights, data):\n", " # third party\n", " import torch\n", - " import torch.nn as nn\n", + " from torch import nn\n", " import torch.nn.functional as F\n", "\n", " class MLP(nn.Module):\n", From f41c6b6c19843d8ef93be209e040f219ecd40e63 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:28:41 +0200 Subject: [PATCH 010/123] Linter --- .../model-training/01-data-scientist-submit-code.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/notebooks/tutorials/model-training/01-data-scientist-submit-code.ipynb b/notebooks/tutorials/model-training/01-data-scientist-submit-code.ipynb index 7b65d033cbc..6779303f7e0 100644 --- a/notebooks/tutorials/model-training/01-data-scientist-submit-code.ipynb +++ b/notebooks/tutorials/model-training/01-data-scientist-submit-code.ipynb @@ -184,8 +184,8 @@ "def mnist_3_linear_layers_torch(mnist_images, mnist_labels):\n", " # third party\n", " import torch\n", - " import torch.nn as nn\n", - " import torch.optim as optim\n", + " from torch import nn\n", + " from torch import optim\n", " from torch.utils.data import TensorDataset\n", "\n", " # Convert NumPy arrays to PyTorch tensors\n", @@ -332,8 +332,8 @@ "def mnist_3_linear_layers_torch(mnist_images, mnist_labels):\n", " # third party\n", " import torch\n", - " import torch.nn as nn\n", - " import torch.optim as optim\n", + " from torch import nn\n", + " from torch import optim\n", " from torch.utils.data import TensorDataset\n", "\n", " # Convert NumPy arrays to PyTorch tensors\n", From 9ad3ec21c44034934fe728da16084de9e4085821 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:30:00 +0200 Subject: [PATCH 011/123] Linter --- .../model-training/03-data-scientist-download-results.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/tutorials/model-training/03-data-scientist-download-results.ipynb b/notebooks/tutorials/model-training/03-data-scientist-download-results.ipynb index de5c4270d2e..a53cdd57c35 100644 --- a/notebooks/tutorials/model-training/03-data-scientist-download-results.ipynb +++ b/notebooks/tutorials/model-training/03-data-scientist-download-results.ipynb @@ -156,7 +156,7 @@ "outputs": [], "source": [ "# third party\n", - "import torch.nn as nn\n", + "from torch import nn\n", "\n", "\n", "class MLP(nn.Module):\n", From 6331932b6884ca75ccd9eabcd25f0bc7f2fc77c8 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:34:52 +0200 Subject: [PATCH 012/123] Linter --- packages/syft/src/syft/service/network/node_peer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index 23c1ffc7057..efb3b35c831 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -146,9 +146,8 @@ def pick_highest_priority_route(self, oldest: bool = True) -> NodeRoute: if oldest: if route.priority < highest_priority_route.priority: highest_priority_route = route - else: - if route.priority > highest_priority_route.priority: - highest_priority_route = route + elif route.priority > highest_priority_route.priority: + highest_priority_route = route return highest_priority_route def update_route(self, route: NodeRoute) -> None: From a845672fc5de7db987db7c8d2a290b1d84e51b62 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:36:33 +0200 Subject: [PATCH 013/123] Linter --- packages/syft/src/syft/service/policy/policy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index e4800f04d6e..3ac30e71089 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -896,12 +896,11 @@ def new_getfile(object: Any) -> Any: # TODO: fix the mypy issue and object.__qualname__ + "." + member.__name__ == member.__qualname__ ): return inspect.getfile(member) - else: - raise TypeError(f"Source for {object!r} not found") + raise TypeError(f"Source for {object!r} not found") def get_code_from_class(policy: type[CustomPolicy]) -> str: - klasses = [inspect.getmro(policy)[0]] # + klasses = [inspect.getmro(policy)[0]] whole_str = "" for klass in klasses: if is_interpreter_jupyter(): From 395a3dd56f690b1c6d7ad0ed1b2095bdb0e482c1 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:37:48 +0200 Subject: [PATCH 014/123] Linter --- packages/syft/src/syft/service/queue/zmq_queue.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 08ff386696e..95d6ef02193 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -788,11 +788,10 @@ def _run(self) -> None: self.reconnect_to_producer() else: logger.error(f"ZMQConsumer invalid command: {command}") - else: - if not self.is_producer_alive(): - logger.info("Producer check-alive timed out. Reconnecting.") - self.reconnect_to_producer() - self.set_producer_alive() + elif not self.is_producer_alive(): + logger.info("Producer check-alive timed out. Reconnecting.") + self.reconnect_to_producer() + self.set_producer_alive() self.send_heartbeat() From d3934fcc2d03d8e3fd6d305b81553f4192c23d2f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:38:55 +0200 Subject: [PATCH 015/123] Linter --- packages/syft/src/syft/store/document_store.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/syft/src/syft/store/document_store.py b/packages/syft/src/syft/store/document_store.py index e71110667c3..c18eb3e4041 100644 --- a/packages/syft/src/syft/store/document_store.py +++ b/packages/syft/src/syft/store/document_store.py @@ -218,11 +218,10 @@ def from_obj(partition_keys: PartitionKeys, obj: SyftObject) -> QueryKeys: pk_value = pk_value() if partition_key.type_list: pk_value = partition_key.extract_list(obj) - else: - if pk_value and not isinstance(pk_value, pk_type): - raise Exception( - f"PartitionKey {pk_value} of type {type(pk_value)} must be {pk_type}." - ) + elif pk_value and not isinstance(pk_value, pk_type): + raise Exception( + f"PartitionKey {pk_value} of type {type(pk_value)} must be {pk_type}." + ) qk = QueryKey(key=pk_key, type_=pk_type, value=pk_value) qks.append(qk) return QueryKeys(qks=qks) From 4f63cd471285a129282b099cb39650dd4ece6200 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:39:49 +0200 Subject: [PATCH 016/123] Linter --- packages/syft/src/syft/types/blob_storage.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/types/blob_storage.py b/packages/syft/src/syft/types/blob_storage.py index a92134f26f7..fb560c2ee65 100644 --- a/packages/syft/src/syft/types/blob_storage.py +++ b/packages/syft/src/syft/types/blob_storage.py @@ -131,11 +131,10 @@ def _iter_lines(self, chunk_size: int = DEFAULT_CHUNK_SIZE) -> Iterator[bytes]: else: pending = None yield from lines + elif pending is None: + pending = chunk else: - if pending is None: - pending = chunk - else: - pending = pending + chunk + pending = pending + chunk if pending is not None: yield pending From 0022ed75d64183b8f2dd3ab629606fb67909f45a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:42:16 +0200 Subject: [PATCH 017/123] Linter --- packages/syft/src/syft/types/syft_object.py | 24 +++++++++------------ 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 10c975bc5f0..f969faabce9 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -595,11 +595,8 @@ def _syft_set_validate_private_attrs_(self, **kwargs: Any) -> None: # Otherwise validate value against the variable annotation check_type(value, var_annotation) setattr(self, attr, value) - else: - if not _is_optional(var_annotation): - raise ValueError( - f"{attr}\n field required (type=value_error.missing)" - ) + elif not _is_optional(var_annotation): + raise ValueError(f"{attr}\n field required (type=value_error.missing)") def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) @@ -768,16 +765,15 @@ def __getattr__(self, item: str) -> Any: raise AttributeError( f"{type(self).__name__!r} object has no attribute {item!r}" ) from exc + elif hasattr(self.__class__, item): + return self.__getattribute__( + item + ) # Raises AttributeError if appropriate else: - if hasattr(self.__class__, item): - return self.__getattribute__( - item - ) # Raises AttributeError if appropriate - else: - # this is the current error - raise AttributeError( - f"{type(self).__name__!r} object has no attribute {item!r}" - ) + # this is the current error + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {item!r}" + ) def short_qual_name(name: str) -> str: From e54a858768da3b93278b30b34425526b562a05d3 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:44:46 +0200 Subject: [PATCH 018/123] Linter --- packages/syft/src/syft/util/schema.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/util/schema.py b/packages/syft/src/syft/util/schema.py index 859efa796b5..10554c3ba6d 100644 --- a/packages/syft/src/syft/util/schema.py +++ b/packages/syft/src/syft/util/schema.py @@ -188,12 +188,11 @@ def resolve_references(json_mappings: dict[str, dict]) -> dict[str, dict]: if _type_str in primitive_mapping.values(): # no issue with primitive types continue - else: + elif _type_str not in json_mappings.keys(): # if we don't have a type yet its because its not supported # lets create an empty type to satisfy the generation process - if _type_str not in json_mappings.keys(): - reference = make_fake_type(_type_str) - new_types[_type_str] = reference + reference = make_fake_type(_type_str) + new_types[_type_str] = reference # if the type is a reference we need to replace its type entry after replace_types[attribute] = { From 3149f50e5ceaf4b03a46e67ef6a729cc6f813c6b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:45:42 +0200 Subject: [PATCH 019/123] Linter --- packages/syft/src/syft/util/util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index d8098f55e1d..44904f2d680 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -324,11 +324,10 @@ def find_available_port( if result_of_check != 0: port_available = True break + elif search: + port += 1 else: - if search: - port += 1 - else: - break + break sock.close() except Exception as e: From 3694436c35402fdfa63852d7880c4be53682af67 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:53:45 +0200 Subject: [PATCH 020/123] Add ignored code PLR2004 --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index 19f5a3a772e..ee22daba371 100644 --- a/ruff.toml +++ b/ruff.toml @@ -24,6 +24,7 @@ ignore = [ "PLR0911", # Too many return statements "PLR0913", # Too many arguments in function definition "PLR0915", # Too many statements + "PLR2004", # Magic value used in comparison ] [lint.per-file-ignores] From 004e9edc5a204e96d20215b63d3962afb2265f4a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 21:56:05 +0200 Subject: [PATCH 021/123] Add ignored code PLR0912 --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index ee22daba371..756508f1519 100644 --- a/ruff.toml +++ b/ruff.toml @@ -22,6 +22,7 @@ ignore = [ "B904", # check for raise statements in exception handlers that lack a from clause "B905", # zip() without an explicit strict= parameter "PLR0911", # Too many return statements + "PLR0912", # Too many branches "PLR0913", # Too many arguments in function definition "PLR0915", # Too many statements "PLR2004", # Magic value used in comparison From 111d513e78ad74e8c826a4bbbbaec7dd32cb0ca5 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 22:01:33 +0200 Subject: [PATCH 022/123] Linter --- packages/grid/backend/grid/core/config.py | 32 +++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index b6f5ddf9067..64b127ea73e 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -108,44 +108,44 @@ def get_emails_enabled(self) -> Self: True if os.getenv("USE_BLOB_STORAGE", "false").lower() == "true" else False ) S3_ENDPOINT: str = os.getenv("S3_ENDPOINT", "seaweedfs") - S3_PORT: int = int(os.getenv("S3_PORT", 8333)) + S3_PORT: int = int(os.getenv("S3_PORT", "8333")) S3_ROOT_USER: str = os.getenv("S3_ROOT_USER", "admin") S3_ROOT_PWD: str | None = os.getenv("S3_ROOT_PWD", "admin") S3_REGION: str = os.getenv("S3_REGION", "us-east-1") S3_PRESIGNED_TIMEOUT_SECS: int = int( - os.getenv("S3_PRESIGNED_TIMEOUT_SECS", 1800) + os.getenv("S3_PRESIGNED_TIMEOUT_SECS", "1800") ) # 30 minutes in seconds - SEAWEED_MOUNT_PORT: int = int(os.getenv("SEAWEED_MOUNT_PORT", 4001)) + SEAWEED_MOUNT_PORT: int = int(os.getenv("SEAWEED_MOUNT_PORT", "4001")) # REDIS_HOST: str = str(os.getenv("REDIS_HOST", "redis")) - # REDIS_PORT: int = int(os.getenv("REDIS_PORT", 6379)) - # REDIS_STORE_DB_ID: int = int(os.getenv("REDIS_STORE_DB_ID", 0)) - # REDIS_LEDGER_DB_ID: int = int(os.getenv("REDIS_LEDGER_DB_ID", 1)) - # STORE_DB_ID: int = int(os.getenv("STORE_DB_ID", 0)) - # LEDGER_DB_ID: int = int(os.getenv("LEDGER_DB_ID", 1)) - # NETWORK_CHECK_INTERVAL: int = int(os.getenv("NETWORK_CHECK_INTERVAL", 60)) - # DOMAIN_CHECK_INTERVAL: int = int(os.getenv("DOMAIN_CHECK_INTERVAL", 60)) + # REDIS_PORT: int = int(os.getenv("REDIS_PORT", "6379")) + # REDIS_STORE_DB_ID: int = int(os.getenv("REDIS_STORE_DB_ID", "0")) + # REDIS_LEDGER_DB_ID: int = int(os.getenv("REDIS_LEDGER_DB_ID", "1")) + # STORE_DB_ID: int = int(os.getenv("STORE_DB_ID", "0")) + # LEDGER_DB_ID: int = int(os.getenv("LEDGER_DB_ID", "1")) + # NETWORK_CHECK_INTERVAL: int = int(os.getenv("NETWORK_CHECK_INTERVAL", "60")) + # DOMAIN_CHECK_INTERVAL: int = int(os.getenv("DOMAIN_CHECK_INTERVAL", "60")) CONTAINER_HOST: str = str(os.getenv("CONTAINER_HOST", "docker")) MONGO_HOST: str = str(os.getenv("MONGO_HOST", "")) - MONGO_PORT: int = int(os.getenv("MONGO_PORT", 27017)) + MONGO_PORT: int = int(os.getenv("MONGO_PORT", "27017")) MONGO_USERNAME: str = str(os.getenv("MONGO_USERNAME", "")) MONGO_PASSWORD: str = str(os.getenv("MONGO_PASSWORD", "")) DEV_MODE: bool = True if os.getenv("DEV_MODE", "false").lower() == "true" else False # ZMQ stuff - QUEUE_PORT: int = int(os.getenv("QUEUE_PORT", 5556)) + QUEUE_PORT: int = int(os.getenv("QUEUE_PORT", "5556")) CREATE_PRODUCER: bool = ( True if os.getenv("CREATE_PRODUCER", "false").lower() == "true" else False ) - N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", 1)) + N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", "1")) SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") - SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", False)) + SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", "False")) CONSUMER_SERVICE_NAME: str | None = os.getenv("CONSUMER_SERVICE_NAME") - INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", True)) + INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", "True")) SMTP_USERNAME: str = os.getenv("SMTP_USERNAME", "") EMAIL_SENDER: str = os.getenv("EMAIL_SENDER", "") SMTP_PASSWORD: str = os.getenv("SMTP_PASSWORD", "") SMTP_TLS: bool = True - SMTP_PORT: int = int(os.getenv("SMTP_PORT", 587)) + SMTP_PORT: int = int(os.getenv("SMTP_PORT", "587")) SMTP_HOST: str = os.getenv("SMTP_HOST", "") TEST_MODE: bool = ( From ee895ddbf15c2155ba0e987a8a0552b432c08772 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 22:05:14 +0200 Subject: [PATCH 023/123] Linter --- packages/syft/src/syft/util/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index 44904f2d680..98f7ea41bf2 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -712,7 +712,7 @@ def autocache( def str_to_bool(bool_str: str | None) -> bool: result = False bool_str = str(bool_str).lower() - if bool_str == "true" or bool_str == "1": + if bool_str in ("true", "1"): result = True return result From ad28841144fe1bd2c02dee7e4fadddcfd71b6832 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 30 Jun 2024 22:06:15 +0200 Subject: [PATCH 024/123] Linter --- packages/syft/src/syft/util/telemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/util/telemetry.py b/packages/syft/src/syft/util/telemetry.py index d03f240a1de..a7f9742ec60 100644 --- a/packages/syft/src/syft/util/telemetry.py +++ b/packages/syft/src/syft/util/telemetry.py @@ -11,7 +11,7 @@ def str_to_bool(bool_str: str | None) -> bool: result = False bool_str = str(bool_str).lower() - if bool_str == "true" or bool_str == "1": + if bool_str in ("true", "1"): result = True return result From 78c5bc17c853439ade156d719fc17a52ea1c8be6 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 04:32:04 +0200 Subject: [PATCH 025/123] Linter --- packages/syft/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/tests/conftest.py b/packages/syft/tests/conftest.py index 2d781f817d7..213e5c47974 100644 --- a/packages/syft/tests/conftest.py +++ b/packages/syft/tests/conftest.py @@ -75,7 +75,7 @@ def is_vscode_discover(): # Pytest hook to set the number of workers for xdist def pytest_xdist_auto_num_workers(config): num = config.option.numprocesses - if num == "auto" or num == "logical": + if num in ("auto", "logical"): return os.cpu_count() return None From 4f434b7b650519ff73f6ca09c71e13122bd0c4dd Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 04:35:53 +0200 Subject: [PATCH 026/123] Add Noqa --- packages/syft/tests/syft/service/action/action_object_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index dd7351f78e4..bc87bedb3d1 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -728,7 +728,7 @@ def test_actionobject_syft_getattr_str(worker, scenario): assert obj.upper() == orig_obj.upper() assert "C" in obj assert "z" not in obj - assert obj[0] == orig_obj[0] + assert obj[0] == orig_obj[0] # noqa assert f"test {obj}" == f"test {orig_obj}" assert obj > "a" assert obj < "zzzz" From 965a195c6a26d7d1b0398f013c866b24c928523d Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 04:40:38 +0200 Subject: [PATCH 027/123] add noqa --- packages/syft/tests/syft/service/action/action_object_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index bc87bedb3d1..0d9d7812499 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -729,12 +729,12 @@ def test_actionobject_syft_getattr_str(worker, scenario): assert "C" in obj assert "z" not in obj assert obj[0] == orig_obj[0] # noqa - assert f"test {obj}" == f"test {orig_obj}" + assert f"test {obj}" == f"test {orig_obj}" assert obj > "a" assert obj < "zzzz" for idx, c in enumerate(obj): assert c == orig_obj[idx] - assert obj[idx] == orig_obj[idx] + assert obj[idx] == orig_obj[idx] # noqa for idx, c in enumerate(orig_obj): assert c == obj[idx] From 60d4ed17faae3dfbe54164a851b6c273e8de467b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 04:42:08 +0200 Subject: [PATCH 028/123] Linter --- packages/syft/src/syft/service/request/request.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 8c5687ac55e..fd775cc34e7 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -1243,13 +1243,12 @@ def __repr_syft_nested__(self) -> str: msg += "to permission RequestStatus.APPROVED." if self.code.nested_codes is None or self.code.nested_codes == {}: # type: ignore msg += " No nested requests" + elif self.nested_solved: + # else: + msg += "

This change requests the following nested functions calls:
" + msg += self.nested_repr() else: - if self.nested_solved: - # else: - msg += "

This change requests the following nested functions calls:
" - msg += self.nested_repr() - else: - msg += " Nested Requests not resolved" + msg += " Nested Requests not resolved" return msg def _repr_markdown_(self, wrap_as_python: bool = True, indent: int = 0) -> str: From 33650c1b9fc9db5ec103a1aeb32d8385c5714655 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 04:48:26 +0200 Subject: [PATCH 029/123] Ignore code PLW0127 Self-assignment of variable --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index 756508f1519..6fbdc866a21 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,6 +26,7 @@ ignore = [ "PLR0913", # Too many arguments in function definition "PLR0915", # Too many statements "PLR2004", # Magic value used in comparison + "PLW0127", # Self-assignment of variable ] [lint.per-file-ignores] From 3e0b64ab69ec5d80af6486b3d1be3e733dbdec1a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 04:50:08 +0200 Subject: [PATCH 030/123] Linter --- packages/syft/tests/syft/service/action/action_object_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index 0d9d7812499..b33f3e3cab6 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -729,7 +729,7 @@ def test_actionobject_syft_getattr_str(worker, scenario): assert "C" in obj assert "z" not in obj assert obj[0] == orig_obj[0] # noqa - assert f"test {obj}" == f"test {orig_obj}" + assert f"test {obj}" == f"test {orig_obj}" assert obj > "a" assert obj < "zzzz" for idx, c in enumerate(obj): From 558b5eff47deb63503f1e78ea801f41f000e3d78 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 05:06:58 +0200 Subject: [PATCH 031/123] Linter --- .../src/syft/service/action/action_service.py | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_service.py b/packages/syft/src/syft/service/action/action_service.py index 10b864732bb..4b6c74c4718 100644 --- a/packages/syft/src/syft/service/action/action_service.py +++ b/packages/syft/src/syft/service/action/action_service.py @@ -1045,19 +1045,16 @@ def filter_twin_kwargs( raise Exception( f"Filter can only use {TwinMode.PRIVATE} or {TwinMode.MOCK}" ) + elif isinstance(v, ActionObject): + filtered[k] = v.syft_action_data + elif ( + isinstance(v, str | int | float | dict | CustomEndpointActionObject) + and allow_python_types + ): + filtered[k] = v else: - if isinstance(v, ActionObject): - filtered[k] = v.syft_action_data - elif ( - isinstance(v, str | int | float | dict | CustomEndpointActionObject) - and allow_python_types - ): - filtered[k] = v - else: - # third party - raise ValueError( - f"unexepected value {v} passed to filtered twin kwargs" - ) + # third party + raise ValueError(f"unexepected value {v} passed to filtered twin kwargs") return filtered From b6d9c0e05061bcb890fbc84a523f6d649b18f48f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:01:59 +0200 Subject: [PATCH 032/123] Linter --- .../src/syft/service/action/action_object.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index b8e22138d26..2cd0213fc1f 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1971,18 +1971,17 @@ def _repr_markdown_(self, wrap_as_python: bool = True, indent: int = 0) -> str: if isinstance(self.syft_action_data_cache, ActionDataEmpty): data_repr_ = self.syft_action_data_repr_ + elif inspect.isclass(self.syft_action_data_cache): + data_repr_ = repr_cls(self.syft_action_data_cache) else: - if inspect.isclass(self.syft_action_data_cache): - data_repr_ = repr_cls(self.syft_action_data_cache) - else: - data_repr_ = ( - self.syft_action_data_cache._repr_markdown_() - if ( - self.syft_action_data_cache is not None - and hasattr(self.syft_action_data_cache, "_repr_markdown_") - ) - else self.syft_action_data_cache.__repr__() + data_repr_ = ( + self.syft_action_data_cache._repr_markdown_() + if ( + self.syft_action_data_cache is not None + and hasattr(self.syft_action_data_cache, "_repr_markdown_") ) + else self.syft_action_data_cache.__repr__() + ) return f"\n**{res}**\n\n{data_repr_}\n" From dd5454d170dada53e4069bb617d4a06072186208 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:03:29 +0200 Subject: [PATCH 033/123] Linter --- packages/syft/src/syft/node/node.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 04851c2b17f..dec0bfe6737 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -1223,9 +1223,8 @@ def handle_api_call_with_unsigned_result( return SyftError( message=f"You sent a {type(api_call)}. This node requires SignedSyftAPICall." ) - else: - if not api_call.is_valid: - return SyftError(message="Your message signature is invalid") + elif not api_call.is_valid: + return SyftError(message="Your message signature is invalid") if api_call.message.node_uid != self.id and check_call_location: return self.forward_message(api_call=api_call) From 6ca9db491c7e514a24bcdef5561b9817036fe6fe Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:05:41 +0200 Subject: [PATCH 034/123] Linter --- packages/syft/src/syft/service/job/job_stash.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/syft/src/syft/service/job/job_stash.py b/packages/syft/src/syft/service/job/job_stash.py index e69c539cb5c..54d40dbc69f 100644 --- a/packages/syft/src/syft/service/job/job_stash.py +++ b/packages/syft/src/syft/service/job/job_stash.py @@ -231,16 +231,15 @@ def progress(self) -> str | None: if self.status in [JobStatus.PROCESSING, JobStatus.COMPLETED]: if self.current_iter is None: return "" - else: - if self.n_iters is not None: - return self.time_remaining_string - # if self.current_iter !=0 - # we can compute the remaining time + elif self.n_iters is not None: + return self.time_remaining_string + # if self.current_iter !=0 + # we can compute the remaining time - # we cannot compute the remaining time - else: - n_iters_str = "?" if self.n_iters is None else str(self.n_iters) - return f"{self.current_iter}/{n_iters_str}" + # we cannot compute the remaining time + else: + n_iters_str = "?" if self.n_iters is None else str(self.n_iters) + return f"{self.current_iter}/{n_iters_str}" else: return "" From a1a3cc723fe42713268634b4aa6ef6970bff20f3 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:06:51 +0200 Subject: [PATCH 035/123] Update job_stash.py --- packages/syft/src/syft/service/job/job_stash.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/job/job_stash.py b/packages/syft/src/syft/service/job/job_stash.py index 54d40dbc69f..dec4849345f 100644 --- a/packages/syft/src/syft/service/job/job_stash.py +++ b/packages/syft/src/syft/service/job/job_stash.py @@ -395,10 +395,9 @@ def logs( # no access if isinstance(self.result, Err): results.append(self.result.value) - else: + elif isinstance(self.result, Err): # add short error - if isinstance(self.result, Err): - results.append(self.result.value) + results.append(self.result.value) if has_permissions: has_storage_permission = api.services.log.has_storage_permission( From 568b173ea546dbbe08a26f7b7590e9ffda81810a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:12:37 +0200 Subject: [PATCH 036/123] Linter --- packages/syft/src/syft/service/sync/diff_state.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index d5f8eb60caf..ef15d730d85 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -1557,8 +1557,7 @@ def from_batch_decision( if share_to_user is None: # job ran by another user if ( - diff.object_type != "Job" - and diff.object_type != "ExecutionOutput" + diff.object_type not in ("Job", "ExecutionOutput") ): raise ValueError( "share_to_user is required to share private data" @@ -1627,8 +1626,7 @@ def add_unignored(self, root_id: UID) -> None: def add_sync_instruction(self, sync_instruction: SyncInstruction) -> None: if ( - sync_instruction.decision == SyncDecision.IGNORE - or sync_instruction.decision == SyncDecision.SKIP + sync_instruction.decision in (SyncDecision.IGNORE, SyncDecision.SKIP) ): return diff = sync_instruction.diff From 47753df40668bafffa4f98b5364292ddee484108 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:17:13 +0200 Subject: [PATCH 037/123] Linter --- packages/syft/src/syft/node/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/run.py b/packages/syft/src/syft/node/run.py index 5d731d48fd5..2b05a239c3e 100644 --- a/packages/syft/src/syft/node/run.py +++ b/packages/syft/src/syft/node/run.py @@ -9,7 +9,7 @@ def str_to_bool(bool_str: str | None) -> bool: result = False bool_str = str(bool_str).lower() - if bool_str == "true" or bool_str == "1": + if bool_str in ("true", "1"): result = True return result From ea76d5c42fb6422d217cfed7fb062f6b286665d9 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:19:22 +0200 Subject: [PATCH 038/123] Linter --- packages/syft/src/syft/service/code/user_code_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 41aba58575b..60d4cdf328b 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -651,8 +651,7 @@ def has_code_permission( self, code_item: UserCode, context: AuthedServiceContext ) -> SyftSuccess | SyftError: if not ( - context.credentials == context.node.verify_key - or context.credentials == code_item.user_verify_key + context.credentials in (context.node.verify_key, code_item.user_verify_key) ): return SyftError( message=f"Code Execution Permission: {context.credentials} denied" From acd482f283deb00333edbd8cbe614f9bd64eb945 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:23:33 +0200 Subject: [PATCH 039/123] Linter --- .../src/syft/service/project/project_service.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/syft/src/syft/service/project/project_service.py b/packages/syft/src/syft/service/project/project_service.py index 3b8cef606ac..708cd5db19c 100644 --- a/packages/syft/src/syft/service/project/project_service.py +++ b/packages/syft/src/syft/service/project/project_service.py @@ -104,18 +104,17 @@ def create_project( ) ) leader_node_peer = peer.ok() - else: + elif project.leader_node_route is not None: # for the leader node, as it does not have route information to itself # we rely on the data scientist to provide the route # the route is then validated by the leader - if project.leader_node_route is not None: - leader_node_peer = project.leader_node_route.validate_with_context( - context=context - ) - else: - return SyftError( - message=f"project {project}'s leader_node_route is None" - ) + leader_node_peer = project.leader_node_route.validate_with_context( + context=context + ) + else: + return SyftError( + message=f"project {project}'s leader_node_route is None" + ) project_obj.leader_node_peer = leader_node_peer From 73782f1564798ed24c337a57a8f49d9d59c00955 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:25:02 +0200 Subject: [PATCH 040/123] Linter --- packages/syft/src/syft/service/request/request.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index fd775cc34e7..8649f8fb990 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -1245,7 +1245,9 @@ def __repr_syft_nested__(self) -> str: msg += " No nested requests" elif self.nested_solved: # else: - msg += "

This change requests the following nested functions calls:
" + msg += ( + "

This change requests the following nested functions calls:
" + ) msg += self.nested_repr() else: msg += " Nested Requests not resolved" From ff3f49cb8d85755a67c882500b8e84f7983318db Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 06:29:10 +0200 Subject: [PATCH 041/123] Linter --- packages/syft/src/syft/service/settings/settings_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index f404c498662..a51a28616ba 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -333,8 +333,7 @@ def welcome_show( ) commands = "" if ( - role.value == ServiceRole.NONE.value - or role.value == ServiceRole.GUEST.value + role.value in (ServiceRole.NONE.value, ServiceRole.GUEST.value) ): commands = GUEST_COMMANDS elif ( From 95401729d952bcf4eb7b155336f2961d4ad9412f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:02:16 +0200 Subject: [PATCH 042/123] Linter --- packages/syft/src/syft/service/sync/diff_state.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index ef15d730d85..3032bb5c126 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -1556,9 +1556,7 @@ def from_batch_decision( if share_private_data: # or diff.object_type == "Job": if share_to_user is None: # job ran by another user - if ( - diff.object_type not in ("Job", "ExecutionOutput") - ): + if diff.object_type not in ("Job", "ExecutionOutput"): raise ValueError( "share_to_user is required to share private data" ) @@ -1625,9 +1623,7 @@ def add_unignored(self, root_id: UID) -> None: self.unignored_batches.add(root_id) def add_sync_instruction(self, sync_instruction: SyncInstruction) -> None: - if ( - sync_instruction.decision in (SyncDecision.IGNORE, SyncDecision.SKIP) - ): + if sync_instruction.decision in (SyncDecision.IGNORE, SyncDecision.SKIP): return diff = sync_instruction.diff From 1be20f4d4394e91e6fce86bdc1c8ecd20c272a48 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:05:45 +0200 Subject: [PATCH 043/123] Update user_code_service.py --- packages/syft/src/syft/service/code/user_code_service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 60d4cdf328b..1b095b8708f 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -650,8 +650,9 @@ def _call( def has_code_permission( self, code_item: UserCode, context: AuthedServiceContext ) -> SyftSuccess | SyftError: - if not ( - context.credentials in (context.node.verify_key, code_item.user_verify_key) + if context.credentials not in ( + context.node.verify_key, + code_item.user_verify_key, ): return SyftError( message=f"Code Execution Permission: {context.credentials} denied" From b05965d1f9341f9c17dcbea1e5bdfb5a7156b7a1 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:07:05 +0200 Subject: [PATCH 044/123] Linter --- packages/syft/src/syft/service/settings/settings_service.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index a51a28616ba..3d9f1dc02c3 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -332,9 +332,7 @@ def welcome_show( else "High Side" ) commands = "" - if ( - role.value in (ServiceRole.NONE.value, ServiceRole.GUEST.value) - ): + if role.value in (ServiceRole.NONE.value, ServiceRole.GUEST.value): commands = GUEST_COMMANDS elif ( role is not None and role.value == ServiceRole.DATA_SCIENTIST.value From dcb9072afad1a64de813cde20889732a52e45c08 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:12:37 +0200 Subject: [PATCH 045/123] Linter --- packages/grid/backend/grid/core/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 64b127ea73e..9ef926ccc86 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -138,7 +138,9 @@ def get_emails_enabled(self) -> Self: ) N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", "1")) SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") - SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", "False")) + SINGLE_CONTAINER_MODE: bool = str_to_bool( + os.getenv("SINGLE_CONTAINER_MODE", "False") + ) CONSUMER_SERVICE_NAME: str | None = os.getenv("CONSUMER_SERVICE_NAME") INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", "True")) SMTP_USERNAME: str = os.getenv("SMTP_USERNAME", "") From 71aebc24c66e87bc8c8e7c769ba4d41189a4062a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:15:43 +0200 Subject: [PATCH 046/123] Update ruff.toml --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index 6fbdc866a21..b6fe0e03832 100644 --- a/ruff.toml +++ b/ruff.toml @@ -21,6 +21,7 @@ select = [ ignore = [ "B904", # check for raise statements in exception handlers that lack a from clause "B905", # zip() without an explicit strict= parameter + "E721", # Already done in another PR "PLR0911", # Too many return statements "PLR0912", # Too many branches "PLR0913", # Too many arguments in function definition From f56e9f560c5934f3544d910346da603c9ae2227f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:17:57 +0200 Subject: [PATCH 047/123] Update ruff.toml --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index b6fe0e03832..be5043254ac 100644 --- a/ruff.toml +++ b/ruff.toml @@ -16,6 +16,7 @@ select = [ "C4", # flake8-comprehensions # "PERF", # perflint "PL", # pylint + # "SIM", # flake8-simplify "UP", # pyupgrade ] ignore = [ From cb56079c61c3221b051853c7c67b73f706b275c0 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:26:15 +0200 Subject: [PATCH 048/123] Linter --- ruff.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index be5043254ac..19803c8039f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -16,7 +16,7 @@ select = [ "C4", # flake8-comprehensions # "PERF", # perflint "PL", # pylint - # "SIM", # flake8-simplify + # "SIM", # flake8-simplify "UP", # pyupgrade ] ignore = [ From 20b99c2a4b1774055f52dc5b6423eeec63cb0d96 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:32:16 +0200 Subject: [PATCH 049/123] Linter - PLW2901 --- packages/grid/seaweedfs/src/automount.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/grid/seaweedfs/src/automount.py b/packages/grid/seaweedfs/src/automount.py index 790e42a0a3e..a78cb7be1bc 100644 --- a/packages/grid/seaweedfs/src/automount.py +++ b/packages/grid/seaweedfs/src/automount.py @@ -20,14 +20,14 @@ def automount(automount_conf: Path, mount_conf_dir: Path) -> None: mounts = config.get("mounts", []) for mount_opts in mounts: - mount_opts = MountOptions(**mount_opts) + mount_opts_val = MountOptions(**mount_opts) try: logger.info( - f"Auto mount type={mount_opts.remote_bucket.type} " - f"bucket={mount_opts.remote_bucket.bucket_name}" + f"Auto mount type={mount_opts_val.remote_bucket.type} " + f"bucket={mount_opts_val.remote_bucket.bucket_name}" ) result = mount_bucket( - mount_opts, + mount_opts_val, conf_dir=mount_conf_dir, overwrite=True, ) From 7fb5195163ea3f1997a552115fcedb4c603c19bb Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 09:36:34 +0200 Subject: [PATCH 050/123] Linter - PLW2901 --- packages/syft/src/syft/client/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index 9c8b244b129..4dc03887873 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -1380,8 +1380,8 @@ def validate_callable_args_and_kwargs( if isinstance(t, _GenericAlias) and type(None) in t.__args__: for v in t.__args__: if issubclass(v, EmailStr): - v = str - check_type(arg, v) # raises Exception + val = str + check_type(arg, val) # raises Exception break # only need one to match else: check_type(arg, t) # raises Exception From 3f9de65d17d715e1c2129547b41c80479e289e8e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 13:45:02 +0200 Subject: [PATCH 051/123] Linter --- .../grid/enclave/attestation/server/cpu_attestation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/grid/enclave/attestation/server/cpu_attestation.py b/packages/grid/enclave/attestation/server/cpu_attestation.py index 356157a2e0f..636111f6de6 100644 --- a/packages/grid/enclave/attestation/server/cpu_attestation.py +++ b/packages/grid/enclave/attestation/server/cpu_attestation.py @@ -8,7 +8,7 @@ def attest_cpu() -> tuple[str, str]: # Fetch report from Micrsoft Attestation library cpu_report = subprocess.run( - ["/app/AttestationClient"], capture_output=True, text=True + ["/app/AttestationClient"], capture_output=True, text=True, check=False ) logger.debug(f"Stdout: {cpu_report.stdout}") logger.debug(f"Stderr: {cpu_report.stderr}") @@ -20,7 +20,10 @@ def attest_cpu() -> tuple[str, str]: # Fetch token from Micrsoft Attestation library cpu_token = subprocess.run( - ["/app/AttestationClient", "-o", "token"], capture_output=True, text=True + ["/app/AttestationClient", "-o", "token"], + capture_output=True, + text=True, + check=False, ) logger.debug(f"Stdout: {cpu_token.stdout}") logger.debug(f"Stderr: {cpu_token.stderr}") From 656f6f391f66fbbe5818837422f9ed619c27429a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 13:46:45 +0200 Subject: [PATCH 052/123] Linter --- packages/grid/seaweedfs/src/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/grid/seaweedfs/src/api.py b/packages/grid/seaweedfs/src/api.py index b3d48bab313..901e20f8015 100644 --- a/packages/grid/seaweedfs/src/api.py +++ b/packages/grid/seaweedfs/src/api.py @@ -79,6 +79,7 @@ def configure_azure(first_res: dict) -> str: bucket_name, container_name, account_key, - ] + ], + check=False, ) return str(res.returncode) From aa68dfba69323d676c944d9f22bfe2f576fce138 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 13:49:12 +0200 Subject: [PATCH 053/123] Linter - PLW2901 --- packages/syft/src/syft/client/domain_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/domain_client.py index c131e5beab5..3ecd9ce5c0f 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/domain_client.py @@ -180,8 +180,8 @@ def get_sync_state(self) -> SyncState | SyftError: for uid, obj in state.objects.items(): if isinstance(obj, ActionObject): - obj = obj.refresh_object(resolve_nested=False) - state.objects[uid] = obj + new_obj = obj.refresh_object(resolve_nested=False) + state.objects[uid] = new_obj return state def apply_state(self, resolved_state: ResolvedSyncState) -> SyftSuccess | SyftError: From 0e243eb999654d956e56a79d0c7dc2de5685c9d0 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 13:59:52 +0200 Subject: [PATCH 054/123] Linter - PLW2901 --- packages/syft/src/syft/protocol/data_protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py index 170c103d7a8..38512828249 100644 --- a/packages/syft/src/syft/protocol/data_protocol.py +++ b/packages/syft/src/syft/protocol/data_protocol.py @@ -497,8 +497,8 @@ def supported_protocols(self) -> list[int | str]: for version, is_supported in self.protocol_support.items(): if is_supported: if version != "dev": - version = int(version) - supported.append(version) + new_version = int(version) + supported.append(new_version) return supported def calculate_supported_protocols(self) -> dict: From c3f5d462c84b388bf74d28e6b00035a4cd4a4a8c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 14:20:45 +0200 Subject: [PATCH 055/123] Bug fix --- packages/grid/seaweedfs/src/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/seaweedfs/src/api.py b/packages/grid/seaweedfs/src/api.py index 901e20f8015..1e6ec295df9 100644 --- a/packages/grid/seaweedfs/src/api.py +++ b/packages/grid/seaweedfs/src/api.py @@ -80,6 +80,6 @@ def configure_azure(first_res: dict) -> str: container_name, account_key, ], - check=False, + check=True, ) return str(res.returncode) From 27f75d5c172bf1e7ed6f2bb11a9de3ed00866f6e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 14:21:51 +0200 Subject: [PATCH 056/123] bugfix --- packages/grid/enclave/attestation/server/cpu_attestation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/grid/enclave/attestation/server/cpu_attestation.py b/packages/grid/enclave/attestation/server/cpu_attestation.py index 636111f6de6..0c416390621 100644 --- a/packages/grid/enclave/attestation/server/cpu_attestation.py +++ b/packages/grid/enclave/attestation/server/cpu_attestation.py @@ -8,7 +8,7 @@ def attest_cpu() -> tuple[str, str]: # Fetch report from Micrsoft Attestation library cpu_report = subprocess.run( - ["/app/AttestationClient"], capture_output=True, text=True, check=False + ["/app/AttestationClient"], capture_output=True, text=True, check=True ) logger.debug(f"Stdout: {cpu_report.stdout}") logger.debug(f"Stderr: {cpu_report.stderr}") @@ -23,7 +23,7 @@ def attest_cpu() -> tuple[str, str]: ["/app/AttestationClient", "-o", "token"], capture_output=True, text=True, - check=False, + check=True, ) logger.debug(f"Stdout: {cpu_token.stdout}") logger.debug(f"Stderr: {cpu_token.stderr}") From 1f8dce78290d6ed654fd0d74a5b91869c944e946 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 14:47:24 +0200 Subject: [PATCH 057/123] revert --- packages/syft/src/syft/protocol/data_protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py index 38512828249..170c103d7a8 100644 --- a/packages/syft/src/syft/protocol/data_protocol.py +++ b/packages/syft/src/syft/protocol/data_protocol.py @@ -497,8 +497,8 @@ def supported_protocols(self) -> list[int | str]: for version, is_supported in self.protocol_support.items(): if is_supported: if version != "dev": - new_version = int(version) - supported.append(new_version) + version = int(version) + supported.append(version) return supported def calculate_supported_protocols(self) -> dict: From 465e049153e7b3637a18cb9274c832f2c4334e68 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 15:02:28 +0200 Subject: [PATCH 058/123] Pylint - PLW2901 --- packages/syft/src/syft/protocol/data_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py index 170c103d7a8..0b3591ad7a8 100644 --- a/packages/syft/src/syft/protocol/data_protocol.py +++ b/packages/syft/src/syft/protocol/data_protocol.py @@ -177,7 +177,7 @@ def build_state(self, stop_key: str | None = None) -> dict: for protocol_number in sorted_dict: object_versions = sorted_dict[protocol_number]["object_versions"] for canonical_name, versions in object_versions.items(): - for version, object_metadata in versions.items(): + for object_metadata in versions.values(): action = object_metadata["action"] version = object_metadata["version"] hash_str = object_metadata["hash"] From ad51ed55d23765f9f63b8bb1a3d0c182a66a1be0 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 15:11:11 +0200 Subject: [PATCH 059/123] Linter PLW2901 --- packages/syft/src/syft/protocol/data_protocol.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py index 0b3591ad7a8..d869c95e9fb 100644 --- a/packages/syft/src/syft/protocol/data_protocol.py +++ b/packages/syft/src/syft/protocol/data_protocol.py @@ -493,13 +493,9 @@ def check_or_stage_protocol(self) -> Result[SyftSuccess, SyftError]: @property def supported_protocols(self) -> list[int | str]: """Returns a list of protocol numbers that are marked as supported.""" - supported = [] - for version, is_supported in self.protocol_support.items(): - if is_supported: - if version != "dev": - version = int(version) - supported.append(version) - return supported + return [int(version) if version != "dev" else version + for version, is_supported in self.protocol_support.items() + if is_supported] def calculate_supported_protocols(self) -> dict: protocol_supported = {} From 2b8d9b106bd47ea873ff9c8876e17e4ab6487365 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 15:13:54 +0200 Subject: [PATCH 060/123] Ruff formatter --- packages/syft/src/syft/protocol/data_protocol.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py index d869c95e9fb..ca06c80e49d 100644 --- a/packages/syft/src/syft/protocol/data_protocol.py +++ b/packages/syft/src/syft/protocol/data_protocol.py @@ -493,9 +493,11 @@ def check_or_stage_protocol(self) -> Result[SyftSuccess, SyftError]: @property def supported_protocols(self) -> list[int | str]: """Returns a list of protocol numbers that are marked as supported.""" - return [int(version) if version != "dev" else version - for version, is_supported in self.protocol_support.items() - if is_supported] + return [ + int(version) if version != "dev" else version + for version, is_supported in self.protocol_support.items() + if is_supported + ] def calculate_supported_protocols(self) -> dict: protocol_supported = {} From 206633cc8313af6255e327380584c35611ae007c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 18:36:33 +0200 Subject: [PATCH 061/123] add noqa for global var --- packages/syft/src/syft/util/autoreload.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/util/autoreload.py b/packages/syft/src/syft/util/autoreload.py index e1f68e45555..d057b3bcb2d 100644 --- a/packages/syft/src/syft/util/autoreload.py +++ b/packages/syft/src/syft/util/autoreload.py @@ -2,7 +2,7 @@ def enable_autoreload() -> None: - global AUTORELOAD_ENABLED + global AUTORELOAD_ENABLED # noqa try: # third party from IPython import get_ipython @@ -18,7 +18,7 @@ def enable_autoreload() -> None: def disable_autoreload() -> None: - global AUTORELOAD_ENABLED + global AUTORELOAD_ENABLED # noqa try: # third party from IPython import get_ipython @@ -32,5 +32,5 @@ def disable_autoreload() -> None: def autoreload_enabled() -> bool: - global AUTORELOAD_ENABLED + global AUTORELOAD_ENABLED # noqa return AUTORELOAD_ENABLED From a683a4ee216b7bfca74c72241fc7cd0338e66f68 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 18:40:23 +0200 Subject: [PATCH 062/123] Update user_settings.py --- packages/syft/src/syft/client/user_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/user_settings.py b/packages/syft/src/syft/client/user_settings.py index 2498f6b1801..abafef55806 100644 --- a/packages/syft/src/syft/client/user_settings.py +++ b/packages/syft/src/syft/client/user_settings.py @@ -22,7 +22,7 @@ def helper(self) -> bool: @helper.setter def helper(self, value: bool) -> None: - global tutorial_mode + global tutorial_mode # noqa tutorial_mode = value From a0695b9c3967e539170e93b2985c8db4b3193957 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 18:41:50 +0200 Subject: [PATCH 063/123] Linter - PLW2901 --- packages/syft/src/syft/types/blob_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/types/blob_storage.py b/packages/syft/src/syft/types/blob_storage.py index fb560c2ee65..57b1ff87d2f 100644 --- a/packages/syft/src/syft/types/blob_storage.py +++ b/packages/syft/src/syft/types/blob_storage.py @@ -124,7 +124,7 @@ def _iter_lines(self, chunk_size: int = DEFAULT_CHUNK_SIZE) -> Iterator[bytes]: for chunk in self.read(stream=True, chunk_size=chunk_size): if b"\n" in chunk: if pending is not None: - chunk = pending + chunk # type: ignore[unreachable] + chunk += pending # type: ignore[unreachable] lines = chunk.splitlines() if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: pending = lines.pop() From d19197686a54272ff5b93fa5e5f2d6f3f16e4b91 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 18:48:03 +0200 Subject: [PATCH 064/123] Linter - PLW2901 --- packages/syft/src/syft/service/sync/diff_state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index 3032bb5c126..e71efba269c 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -1516,9 +1516,9 @@ def _filter( if exclude_types: for exclude_type in exclude_types: if isinstance(exclude_type, type): - exclude_type = exclude_type.__name__ + exclude_type_name = exclude_type.__name__ new_filters.append( - NodeDiffFilter(FilterProperty.TYPE, exclude_type, operator.ne) + NodeDiffFilter(FilterProperty.TYPE, exclude_type_name, operator.ne) ) return self._apply_filters(new_filters, inplace=inplace) From 9052e20102906ae74451979a252e964abbc91075 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 18:56:35 +0200 Subject: [PATCH 065/123] Linter - PLW2901 --- packages/syft/src/syft/service/sync/resolve_widget.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 4a868634df3..3e408d7e657 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -85,7 +85,8 @@ def create_diff_html( for attr, val in properties.items(): status = statuses[attr] - val = val if val is not None else "" + if val is None + val = "" style = f"background-color: {background_colors[status]}; color: {colors[status]}; display: block; white-space: pre-wrap; margin-bottom: 5px;" # noqa: E501 content = html.escape(f"{attr}: {val}") html_str += f"
{content}
" From 768837090eee059858dbcaa4e2a919b62b258d17 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:00:43 +0200 Subject: [PATCH 066/123] Linter - PLW2901 --- packages/syft/src/syft/service/sync/sync_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/sync/sync_service.py b/packages/syft/src/syft/service/sync/sync_service.py index 62885742c5b..c653dbbe826 100644 --- a/packages/syft/src/syft/service/sync/sync_service.py +++ b/packages/syft/src/syft/service/sync/sync_service.py @@ -210,8 +210,8 @@ def sync_items( context, item, new_storage_permissions ) else: - item = self.transform_item(context, item) # type: ignore[unreachable] - res = self.set_object(context, item) + new_item = self.transform_item(context, item) # type: ignore[unreachable] + res = self.set_object(context, new_item) if res.is_ok(): self.add_permissions_for_item(context, item, new_permissions) From 00c2109ddec1d9e58140137189b595dbf6e82073 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:06:15 +0200 Subject: [PATCH 067/123] bugfix missing : --- packages/syft/src/syft/service/sync/resolve_widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 3e408d7e657..9ea0c5014db 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -85,7 +85,7 @@ def create_diff_html( for attr, val in properties.items(): status = statuses[attr] - if val is None + if val is None: val = "" style = f"background-color: {background_colors[status]}; color: {colors[status]}; display: block; white-space: pre-wrap; margin-bottom: 5px;" # noqa: E501 content = html.escape(f"{attr}: {val}") From 6403e446f7f6fbbb7d10a764e3a2067fdc091efe Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:13:18 +0200 Subject: [PATCH 068/123] Update resolve_widget.py --- packages/syft/src/syft/service/sync/resolve_widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 9ea0c5014db..dc4bf40f18d 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -86,7 +86,7 @@ def create_diff_html( for attr, val in properties.items(): status = statuses[attr] if val is None: - val = "" + val = "" # noqa style = f"background-color: {background_colors[status]}; color: {colors[status]}; display: block; white-space: pre-wrap; margin-bottom: 5px;" # noqa: E501 content = html.escape(f"{attr}: {val}") html_str += f"
{content}
" From b7c01b2780d014ec35d3e0c3cc108e7815344ac8 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:19:01 +0200 Subject: [PATCH 069/123] add noqa PLC0205 --- packages/syft/src/syft/types/uid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/types/uid.py b/packages/syft/src/syft/types/uid.py index cd3a0dafba5..2e8815ed790 100644 --- a/packages/syft/src/syft/types/uid.py +++ b/packages/syft/src/syft/types/uid.py @@ -41,7 +41,7 @@ class UID: "value": (lambda x: x.bytes, lambda x: uuid.UUID(bytes=bytes(x))) } - __slots__ = "value" + __slots__ = "value" # noqa PLC0205 value: uuid_type def __init__(self, value: Self | uuid_type | str | bytes | None = None): From 01435159e9185fc497703b139ad2bb3190da023b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:20:13 +0200 Subject: [PATCH 070/123] Update create.py --- packages/syftcli/syftcli/bundle/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syftcli/syftcli/bundle/create.py b/packages/syftcli/syftcli/bundle/create.py index ae3a2893130..f4fff2a509a 100644 --- a/packages/syftcli/syftcli/bundle/create.py +++ b/packages/syftcli/syftcli/bundle/create.py @@ -22,7 +22,7 @@ from ..core.syft_version import InvalidVersion from ..core.syft_version import SyftVersion -__all__ = "create" +__all__ = "create" # noqa PLE0605 DEFAULT_OUTPUT_DIR = Path("~/.syft") From 7ad946c07c3abbbd0d69efa92e50052152a691cc Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:20:53 +0200 Subject: [PATCH 071/123] Update cli.py --- packages/syftcli/syftcli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syftcli/syftcli/cli.py b/packages/syftcli/syftcli/cli.py index cb9761e36f7..aec36c34fb4 100644 --- a/packages/syftcli/syftcli/cli.py +++ b/packages/syftcli/syftcli/cli.py @@ -9,7 +9,7 @@ from syftcli.core.console import console from syftcli.version import __version__ -__all__ = "app" +__all__ = "app" # noqa PLE0605 app = Typer(name="Syft CLI", no_args_is_help=True, pretty_exceptions_show_locals=False) app.add_typer(bundle_cmd, name="bundle") From f489c41db7f277d8cadc009995a6adf7238dc0e1 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:25:19 +0200 Subject: [PATCH 072/123] Adapt noqa --- packages/syft/src/syft/service/sync/resolve_widget.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index dc4bf40f18d..acaf3486592 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -85,8 +85,8 @@ def create_diff_html( for attr, val in properties.items(): status = statuses[attr] - if val is None: - val = "" # noqa + if val is None: # noqa + val = "" style = f"background-color: {background_colors[status]}; color: {colors[status]}; display: block; white-space: pre-wrap; margin-bottom: 5px;" # noqa: E501 content = html.escape(f"{attr}: {val}") html_str += f"
{content}
" From 8fc6030ea3bb90310b07bdcfc7450e52a47ce045 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:32:46 +0200 Subject: [PATCH 073/123] Linter - PLW2901 --- packages/syft/src/syft/service/code/user_code_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 1b095b8708f..8a73352dca0 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -402,9 +402,9 @@ def keep_owned_kwargs( for k, v in kwargs.items(): if isinstance(v, UID): # Jobs have UID kwargs instead of ActionObject - v = action_service.get(context, uid=v) - if v.is_ok(): - v = v.ok() + val = action_service.get(context, uid=v) + if val.is_ok(): + v = val.ok() if ( isinstance(v, ActionObject) and v.syft_client_verify_key == context.credentials From 90e2be99db226d9c12244610fd90dcb1a7d146ef Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:37:07 +0200 Subject: [PATCH 074/123] revert --- packages/syft/src/syft/service/sync/diff_state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index e71efba269c..3032bb5c126 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -1516,9 +1516,9 @@ def _filter( if exclude_types: for exclude_type in exclude_types: if isinstance(exclude_type, type): - exclude_type_name = exclude_type.__name__ + exclude_type = exclude_type.__name__ new_filters.append( - NodeDiffFilter(FilterProperty.TYPE, exclude_type_name, operator.ne) + NodeDiffFilter(FilterProperty.TYPE, exclude_type, operator.ne) ) return self._apply_filters(new_filters, inplace=inplace) From 7c149849c06f45c314f3fd61b351ca03b6863dd9 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:45:03 +0200 Subject: [PATCH 075/123] add noqa --- packages/syft/src/syft/service/sync/resolve_widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index acaf3486592..19c66cccc63 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -86,7 +86,7 @@ def create_diff_html( for attr, val in properties.items(): status = statuses[attr] if val is None: # noqa - val = "" + val = "" # noqa style = f"background-color: {background_colors[status]}; color: {colors[status]}; display: block; white-space: pre-wrap; margin-bottom: 5px;" # noqa: E501 content = html.escape(f"{attr}: {val}") html_str += f"
{content}
" From 265583f17083ea47e36537dc956e8fc10a2dc709 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 19:59:18 +0200 Subject: [PATCH 076/123] Add ignore PLW2901 --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index 19803c8039f..0434081ba09 100644 --- a/ruff.toml +++ b/ruff.toml @@ -29,6 +29,7 @@ ignore = [ "PLR0915", # Too many statements "PLR2004", # Magic value used in comparison "PLW0127", # Self-assignment of variable + "PLW2901", # `for` loop variable overwritten by assignment target ] [lint.per-file-ignores] From 83af00939637c9597dcb9a85be4f85c73680cd32 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:03:32 +0200 Subject: [PATCH 077/123] revert --- packages/syft/src/syft/service/sync/resolve_widget.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 19c66cccc63..4a868634df3 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -85,8 +85,7 @@ def create_diff_html( for attr, val in properties.items(): status = statuses[attr] - if val is None: # noqa - val = "" # noqa + val = val if val is not None else "" style = f"background-color: {background_colors[status]}; color: {colors[status]}; display: block; white-space: pre-wrap; margin-bottom: 5px;" # noqa: E501 content = html.escape(f"{attr}: {val}") html_str += f"
{content}
" From 4b14e434f1b52d748018ff69a8353c43534a7a42 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:07:56 +0200 Subject: [PATCH 078/123] Linter - PLE1205 --- packages/syft/src/syft/service/action/action_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 2cd0213fc1f..037cfb2b0a1 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1840,7 +1840,7 @@ def wrapper(_self: Any, *args: Any, **kwargs: Any) -> Any: try: wrapper.__doc__ = original_func.__doc__ logger.debug( - "Found original signature for ", + "Found original signature for %s: %s", name, inspect.signature(original_func), ) From 5d8f2be55f2bc452f4e658c67b0b7a5cf97acdc3 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:10:24 +0200 Subject: [PATCH 079/123] Linter - PLE1205 --- packages/syft/src/syft/service/request/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 8649f8fb990..2d06db14f82 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -165,7 +165,7 @@ def _run( ) if apply: logger.debug( - "ADDING PERMISSION", requesting_permission_action_obj, id_action + "ADDING PERMISSION: %s, %s", requesting_permission_action_obj, id_action ) action_store.add_permission(requesting_permission_action_obj) blob_storage_service.stash.add_permission( From 9d160b018025f3c3e253d40b1802921c4e2cef1c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:14:38 +0200 Subject: [PATCH 080/123] Ruff formatter --- packages/syft/src/syft/service/request/request.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 2d06db14f82..58d3700485c 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -165,7 +165,9 @@ def _run( ) if apply: logger.debug( - "ADDING PERMISSION: %s, %s", requesting_permission_action_obj, id_action + "ADDING PERMISSION: %s, %s", + requesting_permission_action_obj, + id_action, ) action_store.add_permission(requesting_permission_action_obj) blob_storage_service.stash.add_permission( From 8bcecb207b890a13215e59c23fe55345831e0b20 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:16:47 +0200 Subject: [PATCH 081/123] Linter - PLR0124 --- packages/syft/tests/syft/uid_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/tests/syft/uid_test.py b/packages/syft/tests/syft/uid_test.py index f2cfd2b9a3f..0d293a7128f 100644 --- a/packages/syft/tests/syft/uid_test.py +++ b/packages/syft/tests/syft/uid_test.py @@ -48,7 +48,7 @@ def test_uid_comparison() -> None: uid2 = UID() # first do some basic checks - assert uid1 == uid1 + # assert uid1 == uid1 assert uid1 != uid2 # since the comparison should be based on the underlying value From 567bea7228fd22443a98766d5e48d9500a343925 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:20:01 +0200 Subject: [PATCH 082/123] Lint - PLE1205 --- .../src/syft/util/notebook_ui/components/tabulator_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index 24b65f0b8f0..2b971426006 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -157,7 +157,7 @@ def build_tabulator_table_with_data( uid, table_data, table_metadata, max_height, pagination, header_sort ) except Exception as e: - logger.debug("error building table", e) + logger.debug("Error building table: %s", e) return None From f8e343ef8fd51f346426aea1be25f756c268f34b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:23:45 +0200 Subject: [PATCH 083/123] Linter - PLW0129 --- packages/grid/seaweedfs/tests/mount_cmd_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/seaweedfs/tests/mount_cmd_test.py b/packages/grid/seaweedfs/tests/mount_cmd_test.py index 4489989cc34..34a491217fa 100644 --- a/packages/grid/seaweedfs/tests/mount_cmd_test.py +++ b/packages/grid/seaweedfs/tests/mount_cmd_test.py @@ -70,7 +70,7 @@ def test_supervisord_conf() -> None: conf = create_supervisord_conf(args) assert "command=echo hello" in conf assert "priority=5" in conf - assert f"[program:{args.name}]" + # assert f"[program:{args.name}]" Asserting on a non-empty string literal will always pass def test_supervisord_incomplete_args() -> None: From 626ff896859e67f622af2226d0169cdba5e86e94 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:27:50 +0200 Subject: [PATCH 084/123] Linter - PLR1704 --- packages/syft/tests/syft/worker_pool/worker_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/syft/tests/syft/worker_pool/worker_test.py b/packages/syft/tests/syft/worker_pool/worker_test.py index c6935006a61..be8f49502c7 100644 --- a/packages/syft/tests/syft/worker_pool/worker_test.py +++ b/packages/syft/tests/syft/worker_pool/worker_test.py @@ -54,9 +54,9 @@ def test_syft_worker(worker: Worker): if pool.name == pool_name: worker_pool = pool assert len(worker_pool.worker_list) == num_workers - for i, worker in enumerate(worker_pool.workers): - assert worker.name == pool_name + "-" + str(i + 1) - assert isinstance(worker.logs, str) - assert worker.worker_pool_name == pool_name - assert isinstance(worker.created_at, DateTime) - assert worker.job_id is None + for i, worker_val in enumerate(worker_pool.workers): + assert worker_val.name == pool_name + "-" + str(i + 1) + assert isinstance(worker_val.logs, str) + assert worker_val.worker_pool_name == pool_name + assert isinstance(worker_val.created_at, DateTime) + assert worker_val.job_id is None From 0cabafc6d4a9049990544283377479ebdbd948c1 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 1 Jul 2024 20:30:06 +0200 Subject: [PATCH 085/123] Linter - PLR1704 --- scripts/print_fd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/print_fd.py b/scripts/print_fd.py index 4d9944dfc10..78928a8bfa8 100644 --- a/scripts/print_fd.py +++ b/scripts/print_fd.py @@ -76,8 +76,8 @@ def main(pid=None, verbose=False): lsof_output = run_lsof_for_pid(pid) if pid else run_lsof() files_by_pid = parse_lsof_output(lsof_output, verbose) - for pid, files in files_by_pid.items(): - print(f"PID {pid} open files:") + for pid_key, files in files_by_pid.items(): + print(f"PID {pid_key} open files:") for file in files: print(f" {file['File Path']} ({file['FD Type']} - {file['FD Info']})") From ecbde9ee90997e866a4385aeb199caa96c7f762f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 11 Jul 2024 08:49:43 +0200 Subject: [PATCH 086/123] update ruff to version 0.5.1 --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 91d1926c03c..dadd071751f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-ast exclude: ^(packages/syft/tests/mongomock) @@ -84,7 +84,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.5.0" + rev: "v0.5.1" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] From 7b9a74368d7614725ecaf2c9bdfd2de20241d4a4 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 11 Jul 2024 09:09:48 +0200 Subject: [PATCH 087/123] Lint - PLW1508 --- packages/grid/backend/grid/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 25e3b7e3bdb..3a04cca3aea 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -157,7 +157,7 @@ def get_emails_enabled(self) -> Self: ASSOCIATION_REQUEST_AUTO_APPROVAL: bool = str_to_bool( os.getenv("ASSOCIATION_REQUEST_AUTO_APPROVAL", "False") ) - MIN_SIZE_BLOB_STORAGE_MB: int = int(os.getenv("MIN_SIZE_BLOB_STORAGE_MB", 16)) + MIN_SIZE_BLOB_STORAGE_MB: int = int(os.getenv("MIN_SIZE_BLOB_STORAGE_MB", "16")) REVERSE_TUNNEL_ENABLED: bool = str_to_bool( os.getenv("REVERSE_TUNNEL_ENABLED", "false") ) From 461a501a1584e24ac99ac8d6cebf3efd46fcd8b0 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 11 Jul 2024 09:11:46 +0200 Subject: [PATCH 088/123] Lint - PLW1508 --- packages/syft/src/syft/node/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 15ae810618d..81ec5aa6550 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -471,7 +471,7 @@ def init_blob_storage(self, config: BlobStorageConfig | None = None) -> None: ) config_ = OnDiskBlobStorageConfig( client_config=client_config, - min_blob_size=os.getenv("MIN_SIZE_BLOB_STORAGE_MB", 16), + min_blob_size=int(os.getenv("MIN_SIZE_BLOB_STORAGE_MB", "16")), ) else: config_ = config From a5be9cede88c6d7be8a040f62fbe6f4ddc1f4502 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 11 Jul 2024 09:13:19 +0200 Subject: [PATCH 089/123] add noqa: PLW0602 --- packages/syft/tests/syft/users/user_code_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/tests/syft/users/user_code_test.py b/packages/syft/tests/syft/users/user_code_test.py index 2b1e206c7a4..b5ecbbdf857 100644 --- a/packages/syft/tests/syft/users/user_code_test.py +++ b/packages/syft/tests/syft/users/user_code_test.py @@ -445,7 +445,7 @@ def test_submit_code_with_global_var(guest_client: DomainClient) -> None: input_policy=sy.ExactMatch(), output_policy=sy.SingleExecutionExactOutput() ) def mock_syft_func_with_global(): - global x + global x # noqa: PLW0602 return x res = guest_client.code.submit(mock_syft_func_with_global) @@ -453,7 +453,7 @@ def mock_syft_func_with_global(): @sy.syft_function_single_use() def mock_syft_func_single_use_with_global(): - global x + global x # noqa: PLW0602 return x res = guest_client.code.submit(mock_syft_func_single_use_with_global) From da2452155b14fcbf97090adce01a2ed1be2d79c8 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sat, 20 Jul 2024 07:04:27 +0200 Subject: [PATCH 090/123] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dadd071751f..6724da0518c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,7 +84,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.5.1" + rev: "v0.5.3" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] From effd2ad123aef842902d298724f2aa5daa316416 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sat, 20 Jul 2024 07:34:17 +0200 Subject: [PATCH 091/123] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6724da0518c..ac20b4e3a10 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -94,7 +94,7 @@ repos: types_or: [python, pyi, jupyter] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.11.0 hooks: - id: mypy name: "mypy: syft-cli" @@ -119,7 +119,7 @@ repos: ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.11.0 hooks: - id: mypy name: "mypy: grid" @@ -144,7 +144,7 @@ repos: ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.11.0 hooks: - id: mypy name: "mypy: syft" From a9654aab096003e7f6dc596b9841549549956b38 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 10:59:46 +0200 Subject: [PATCH 092/123] Update ruff to version 0.5.4 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac20b4e3a10..b38a0b209c3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,7 +84,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.5.3" + rev: "v0.5.4" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] From e4bba64a457b14d0e7953bf053b62a5e2940ca8f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 14:22:37 +0200 Subject: [PATCH 093/123] Update and rename domain_client.py to datasite_client.py --- .../{domain_client.py => datasite_client.py} | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) rename packages/syft/src/syft/client/{domain_client.py => datasite_client.py} (91%) diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/datasite_client.py similarity index 91% rename from packages/syft/src/syft/client/domain_client.py rename to packages/syft/src/syft/client/datasite_client.py index 15f22bfbfbf..aa7c2ce2894 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/datasite_client.py @@ -16,7 +16,7 @@ from tqdm import tqdm # relative -from ..abstract_node import NodeSideType +from ..abstract_server import ServerSideType from ..serde.serializable import serializable from ..service.action.action_object import ActionObject from ..service.code_history.code_history import CodeHistoriesDict @@ -41,14 +41,14 @@ from .client import SyftClient from .client import login from .client import login_as_guest -from .connection import NodeConnection +from .connection import ServerConnection from .protocol import SyftProtocol logger = logging.getLogger(__name__) if TYPE_CHECKING: # relative - from ..orchestra import NodeHandle + from ..orchestra import ServerHandle from ..service.project.project import Project @@ -90,10 +90,10 @@ def add_default_uploader( return obj -@serializable() -class DomainClient(SyftClient): +@serializable(canonical_name="DatasiteClient", version=1) +class DatasiteClient(SyftClient): def __repr__(self) -> str: - return f"" + return f"" def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError: # relative @@ -108,22 +108,22 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError: asset = dataset.asset_list[i] dataset.asset_list[i] = add_default_uploader(user, asset) - dataset._check_asset_must_contain_mock() + # dataset._check_asset_must_contain_mock() dataset_size: float = 0.0 # TODO: Refactor so that object can also be passed to generate warnings - self.api.connection = cast(NodeConnection, self.api.connection) + self.api.connection = cast(ServerConnection, self.api.connection) - metadata = self.api.connection.get_node_metadata(self.api.signing_key) + metadata = self.api.connection.get_server_metadata(self.api.signing_key) if ( metadata.show_warnings - and metadata.node_side_type == NodeSideType.HIGH_SIDE.value + and metadata.server_side_type == ServerSideType.HIGH_SIDE.value ): message = ( "You're approving a request on " - f"{metadata.node_side_type} side {metadata.node_type} " + f"{metadata.server_side_type} side {metadata.server_type} " "which may host datasets with private information." ) prompt_warning_message(message=message, confirm=True) @@ -137,7 +137,7 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError: twin = TwinObject( private_obj=ActionObject.from_obj(asset.data), mock_obj=ActionObject.from_obj(asset.mock), - syft_node_location=self.id, + syft_server_location=self.id, syft_client_verify_key=self.verify_key, ) res = twin._save_to_blob_storage(allow_empty=contains_empty) @@ -157,7 +157,7 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError: return response asset.action_id = twin.id - asset.node_uid = self.id + asset.server_uid = self.id dataset_size += get_mb_size(asset.data) # Update the progress bar and set the dynamic description @@ -172,7 +172,7 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError: def refresh(self) -> None: if self.credentials: - self._fetch_node_metadata(self.credentials) + self._fetch_server_metadata(self.credentials) if self._api and self._api.refresh_api_callback: self._api.refresh_api_callback() @@ -184,8 +184,8 @@ def get_sync_state(self) -> SyncState | SyftError: for uid, obj in state.objects.items(): if isinstance(obj, ActionObject): - new_obj = obj.refresh_object(resolve_nested=False) - state.objects[uid] = new_obj + obj = obj.refresh_object(resolve_nested=False) + state.objects[uid] = obj return state def apply_state(self, resolved_state: ResolvedSyncState) -> SyftSuccess | SyftError: @@ -288,7 +288,7 @@ def connect_to_gateway( via_client: SyftClient | None = None, url: str | None = None, port: int | None = None, - handle: NodeHandle | None = None, # noqa: F821 + handle: ServerHandle | None = None, # noqa: F821 email: str | None = None, password: str | None = None, protocol: str | SyftProtocol = SyftProtocol.HTTP, @@ -319,7 +319,7 @@ def connect_to_gateway( if self.metadata: return SyftSuccess( message=( - f"Connected {self.metadata.node_type} " + f"Connected {self.metadata.server_type} " f"'{self.metadata.name}' to gateway '{client.name}'. " f"{res.message}" ) @@ -334,10 +334,12 @@ def _get_service_by_name_if_exists(self, name: str) -> APIModule | None: return getattr(self.api.services, name) return None - def set_node_side_type_dangerous( - self, node_side_type: str + def set_server_side_type_dangerous( + self, server_side_type: str ) -> Result[SyftSuccess, SyftError]: - return self.api.services.settings.set_node_side_type_dangerous(node_side_type) + return self.api.services.settings.set_server_side_type_dangerous( + server_side_type + ) @property def data_subject_registry(self) -> APIModule | None: @@ -421,10 +423,10 @@ def load_migration_data(self, path: str | Path) -> SyftSuccess | SyftError: return migration_data migration_data._set_obj_location_(self.id, self.verify_key) - if self.id != migration_data.node_uid: + if self.id != migration_data.server_uid: return SyftError( - message=f"This Migration data is not for this node. Expected node id {self.id}, " - f"got {migration_data.node_uid}" + message=f"This Migration data is not for this server. Expected server id {self.id}, " + f"got {migration_data.server_uid}" ) if migration_data.signing_key.verify_key != self.verify_key: @@ -464,7 +466,7 @@ def _repr_html_(self) -> str: if isinstance(obj, SyftError): return obj.message updated_template_str = Template(obj.text).safe_substitute( - node_url=getattr(self.connection, "url", None) + server_url=getattr(self.connection, "url", None) ) # If it's a markdown structured file if not isinstance(obj, HTMLObject): From 159dd65a2c159e99300763901daa8907695cade6 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 14:28:43 +0200 Subject: [PATCH 094/123] Update run.py --- packages/syft/src/syft/node/run.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/node/run.py b/packages/syft/src/syft/node/run.py index 2b05a239c3e..9fc4878a983 100644 --- a/packages/syft/src/syft/node/run.py +++ b/packages/syft/src/syft/node/run.py @@ -2,26 +2,30 @@ import argparse # relative -from ..orchestra import NodeHandle from ..orchestra import Orchestra +from ..orchestra import ServerHandle def str_to_bool(bool_str: str | None) -> bool: result = False bool_str = str(bool_str).lower() - if bool_str in ("true", "1"): + if bool_str == "true" or bool_str == "1": result = True return result -def run() -> NodeHandle | None: +def run() -> ServerHandle | None: parser = argparse.ArgumentParser() parser.add_argument("command", help="command: launch", type=str, default="none") parser.add_argument( - "--name", help="node name", type=str, default="syft-node", dest="name" + "--name", help="server name", type=str, default="syft-server", dest="name" ) parser.add_argument( - "--node-type", help="node type", type=str, default="python", dest="node_type" + "--server-type", + help="server type", + type=str, + default="python", + dest="server_type", ) parser.add_argument( "--host", @@ -80,9 +84,9 @@ def run() -> NodeHandle | None: args.tail = str_to_bool(args.tail) args.cmd = str_to_bool(args.cmd) - node = Orchestra.launch( + server = Orchestra.launch( name=args.name, - node_type=args.node_type, + server_type=args.server_type, host=args.host, port=args.port, dev_mode=args.dev_mode, @@ -92,5 +96,5 @@ def run() -> NodeHandle | None: tail=args.tail, ) if not args.tail: - return node + return server return None From 65eeab5340de4334822dab0527a231feaaf8bd1c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 14:30:46 +0200 Subject: [PATCH 095/123] Update config.py --- packages/grid/backend/grid/core/config.py | 42 +++++++++++------------ 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 3a04cca3aea..0d7d090743a 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -97,57 +97,55 @@ def get_emails_enabled(self) -> Self: DEFAULT_ROOT_PASSWORD: str = "changethis" USERS_OPEN_REGISTRATION: bool = False - NODE_NAME: str = "default_node_name" + SERVER_NAME: str = "default_server_name" STREAM_QUEUE: bool = False - NODE_TYPE: str = "domain" + SERVER_TYPE: str = "datasite" OPEN_REGISTRATION: bool = True - # DOMAIN_ASSOCIATION_REQUESTS_AUTOMATICALLY_ACCEPTED: bool = True + # DATASITE_ASSOCIATION_REQUESTS_AUTOMATICALLY_ACCEPTED: bool = True USE_BLOB_STORAGE: bool = ( True if os.getenv("USE_BLOB_STORAGE", "false").lower() == "true" else False ) S3_ENDPOINT: str = os.getenv("S3_ENDPOINT", "seaweedfs") - S3_PORT: int = int(os.getenv("S3_PORT", "8333")) + S3_PORT: int = int(os.getenv("S3_PORT", 8333)) S3_ROOT_USER: str = os.getenv("S3_ROOT_USER", "admin") S3_ROOT_PWD: str | None = os.getenv("S3_ROOT_PWD", "admin") S3_REGION: str = os.getenv("S3_REGION", "us-east-1") S3_PRESIGNED_TIMEOUT_SECS: int = int( - os.getenv("S3_PRESIGNED_TIMEOUT_SECS", "1800") + os.getenv("S3_PRESIGNED_TIMEOUT_SECS", 1800) ) # 30 minutes in seconds - SEAWEED_MOUNT_PORT: int = int(os.getenv("SEAWEED_MOUNT_PORT", "4001")) + SEAWEED_MOUNT_PORT: int = int(os.getenv("SEAWEED_MOUNT_PORT", 4001)) # REDIS_HOST: str = str(os.getenv("REDIS_HOST", "redis")) - # REDIS_PORT: int = int(os.getenv("REDIS_PORT", "6379")) - # REDIS_STORE_DB_ID: int = int(os.getenv("REDIS_STORE_DB_ID", "0")) - # REDIS_LEDGER_DB_ID: int = int(os.getenv("REDIS_LEDGER_DB_ID", "1")) - # STORE_DB_ID: int = int(os.getenv("STORE_DB_ID", "0")) - # LEDGER_DB_ID: int = int(os.getenv("LEDGER_DB_ID", "1")) - # NETWORK_CHECK_INTERVAL: int = int(os.getenv("NETWORK_CHECK_INTERVAL", "60")) - # DOMAIN_CHECK_INTERVAL: int = int(os.getenv("DOMAIN_CHECK_INTERVAL", "60")) + # REDIS_PORT: int = int(os.getenv("REDIS_PORT", 6379)) + # REDIS_STORE_DB_ID: int = int(os.getenv("REDIS_STORE_DB_ID", 0)) + # REDIS_LEDGER_DB_ID: int = int(os.getenv("REDIS_LEDGER_DB_ID", 1)) + # STORE_DB_ID: int = int(os.getenv("STORE_DB_ID", 0)) + # LEDGER_DB_ID: int = int(os.getenv("LEDGER_DB_ID", 1)) + # NETWORK_CHECK_INTERVAL: int = int(os.getenv("NETWORK_CHECK_INTERVAL", 60)) + # DATASITE_CHECK_INTERVAL: int = int(os.getenv("DATASITE_CHECK_INTERVAL", 60)) CONTAINER_HOST: str = str(os.getenv("CONTAINER_HOST", "docker")) MONGO_HOST: str = str(os.getenv("MONGO_HOST", "")) - MONGO_PORT: int = int(os.getenv("MONGO_PORT", "27017")) + MONGO_PORT: int = int(os.getenv("MONGO_PORT", 27017)) MONGO_USERNAME: str = str(os.getenv("MONGO_USERNAME", "")) MONGO_PASSWORD: str = str(os.getenv("MONGO_PASSWORD", "")) DEV_MODE: bool = True if os.getenv("DEV_MODE", "false").lower() == "true" else False # ZMQ stuff - QUEUE_PORT: int = int(os.getenv("QUEUE_PORT", "5556")) + QUEUE_PORT: int = int(os.getenv("QUEUE_PORT", 5556)) CREATE_PRODUCER: bool = ( True if os.getenv("CREATE_PRODUCER", "false").lower() == "true" else False ) - N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", "1")) + N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", 1)) SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") - SINGLE_CONTAINER_MODE: bool = str_to_bool( - os.getenv("SINGLE_CONTAINER_MODE", "False") - ) + SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", False)) CONSUMER_SERVICE_NAME: str | None = os.getenv("CONSUMER_SERVICE_NAME") - INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", "True")) + INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", True)) SMTP_USERNAME: str = os.getenv("SMTP_USERNAME", "") EMAIL_SENDER: str = os.getenv("EMAIL_SENDER", "") SMTP_PASSWORD: str = os.getenv("SMTP_PASSWORD", "") SMTP_TLS: bool = True - SMTP_PORT: int = int(os.getenv("SMTP_PORT", "587")) + SMTP_PORT: int = int(os.getenv("SMTP_PORT", 587)) SMTP_HOST: str = os.getenv("SMTP_HOST", "") TEST_MODE: bool = ( @@ -157,7 +155,7 @@ def get_emails_enabled(self) -> Self: ASSOCIATION_REQUEST_AUTO_APPROVAL: bool = str_to_bool( os.getenv("ASSOCIATION_REQUEST_AUTO_APPROVAL", "False") ) - MIN_SIZE_BLOB_STORAGE_MB: int = int(os.getenv("MIN_SIZE_BLOB_STORAGE_MB", "16")) + MIN_SIZE_BLOB_STORAGE_MB: int = int(os.getenv("MIN_SIZE_BLOB_STORAGE_MB", 16)) REVERSE_TUNNEL_ENABLED: bool = str_to_bool( os.getenv("REVERSE_TUNNEL_ENABLED", "false") ) From 2cef05b6ef19c69f6b22b468f4f7133ad5fc876d Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 17:50:13 +0200 Subject: [PATCH 096/123] Delete packages/syft/src/syft/node directory --- packages/syft/src/syft/node/__init__.py | 0 packages/syft/src/syft/node/credentials.py | 103 - packages/syft/src/syft/node/domain.py | 8 - packages/syft/src/syft/node/enclave.py | 11 - packages/syft/src/syft/node/gateway.py | 11 - packages/syft/src/syft/node/node.py | 1771 ----------------- packages/syft/src/syft/node/routes.py | 273 --- packages/syft/src/syft/node/run.py | 100 - packages/syft/src/syft/node/server.py | 328 --- .../syft/src/syft/node/service_registry.py | 135 -- packages/syft/src/syft/node/utils.py | 38 - packages/syft/src/syft/node/worker.py | 8 - .../syft/src/syft/node/worker_settings.py | 52 - 13 files changed, 2838 deletions(-) delete mode 100644 packages/syft/src/syft/node/__init__.py delete mode 100644 packages/syft/src/syft/node/credentials.py delete mode 100644 packages/syft/src/syft/node/domain.py delete mode 100644 packages/syft/src/syft/node/enclave.py delete mode 100644 packages/syft/src/syft/node/gateway.py delete mode 100644 packages/syft/src/syft/node/node.py delete mode 100644 packages/syft/src/syft/node/routes.py delete mode 100644 packages/syft/src/syft/node/run.py delete mode 100644 packages/syft/src/syft/node/server.py delete mode 100644 packages/syft/src/syft/node/service_registry.py delete mode 100644 packages/syft/src/syft/node/utils.py delete mode 100644 packages/syft/src/syft/node/worker.py delete mode 100644 packages/syft/src/syft/node/worker_settings.py diff --git a/packages/syft/src/syft/node/__init__.py b/packages/syft/src/syft/node/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/syft/src/syft/node/credentials.py b/packages/syft/src/syft/node/credentials.py deleted file mode 100644 index 35380a11ae3..00000000000 --- a/packages/syft/src/syft/node/credentials.py +++ /dev/null @@ -1,103 +0,0 @@ -# future -from __future__ import annotations - -# stdlib -from typing import Any - -# third party -from nacl.encoding import HexEncoder -from nacl.signing import SigningKey -from nacl.signing import VerifyKey -from pydantic import field_validator - -# relative -from ..serde.serializable import serializable -from ..types.base import SyftBaseModel - -SIGNING_KEY_FOR = "Corresponding Public Key" - - -@serializable() -class SyftVerifyKey(SyftBaseModel): - verify_key: VerifyKey - - def __init__(self, verify_key: str | VerifyKey): - if isinstance(verify_key, str): - verify_key = VerifyKey(bytes.fromhex(verify_key)) - super().__init__(verify_key=verify_key) - - def __str__(self) -> str: - return self.verify_key.encode(encoder=HexEncoder).decode("utf-8") - - @staticmethod - def from_string(key_str: str) -> SyftVerifyKey: - return SyftVerifyKey(verify_key=VerifyKey(bytes.fromhex(key_str))) - - @property - def verify(self) -> str: - return str(self) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, SyftVerifyKey): - return False - return self.verify_key == other.verify_key - - def __repr__(self) -> str: - return str(self) - - def __hash__(self) -> int: - return hash(self.verify_key) - - -@serializable() -class SyftSigningKey(SyftBaseModel): - signing_key: SigningKey - - @field_validator("signing_key", mode="before") - @classmethod - def make_signing_key(cls, v: Any) -> Any: - return SigningKey(bytes.fromhex(v)) if isinstance(v, str) else v - - @property - def verify_key(self) -> SyftVerifyKey: - return SyftVerifyKey(verify_key=self.signing_key.verify_key) - - def __str__(self) -> str: - return self.signing_key.encode(encoder=HexEncoder).decode("utf-8") - - @staticmethod - def generate() -> SyftSigningKey: - return SyftSigningKey(signing_key=SigningKey.generate()) - - @staticmethod - def from_string(key_str: str) -> SyftSigningKey: - return SyftSigningKey(signing_key=SigningKey(bytes.fromhex(key_str))) - - def __repr__(self) -> str: - return f"<{SIGNING_KEY_FOR}: {self.verify}>" - - def _coll_repr_(self) -> dict[str, str]: - return { - SIGNING_KEY_FOR: self.verify, - } - - @property - def verify(self) -> str: - return str(self.verify_key) - - def __hash__(self) -> int: - return hash(self.signing_key) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, SyftSigningKey): - return False - return self.signing_key == other.signing_key - - -SyftCredentials = SyftVerifyKey | SyftSigningKey - - -@serializable() -class UserLoginCredentials(SyftBaseModel): - email: str - password: str diff --git a/packages/syft/src/syft/node/domain.py b/packages/syft/src/syft/node/domain.py deleted file mode 100644 index 7070e2de756..00000000000 --- a/packages/syft/src/syft/node/domain.py +++ /dev/null @@ -1,8 +0,0 @@ -# relative -from ..serde.serializable import serializable -from .node import Node - - -@serializable(without=["queue_manager"]) -class Domain(Node): - pass diff --git a/packages/syft/src/syft/node/enclave.py b/packages/syft/src/syft/node/enclave.py deleted file mode 100644 index 7ef3505ae93..00000000000 --- a/packages/syft/src/syft/node/enclave.py +++ /dev/null @@ -1,11 +0,0 @@ -# relative -from ..abstract_node import NodeType -from ..serde.serializable import serializable -from .node import Node - - -@serializable() -class Enclave(Node): - def post_init(self) -> None: - self.node_type = NodeType.ENCLAVE - super().post_init() diff --git a/packages/syft/src/syft/node/gateway.py b/packages/syft/src/syft/node/gateway.py deleted file mode 100644 index fc8be3c975e..00000000000 --- a/packages/syft/src/syft/node/gateway.py +++ /dev/null @@ -1,11 +0,0 @@ -# relative -from ..abstract_node import NodeType -from ..serde.serializable import serializable -from .node import Node - - -@serializable() -class Gateway(Node): - def post_init(self) -> None: - self.node_type = NodeType.GATEWAY - super().post_init() diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py deleted file mode 100644 index 81ec5aa6550..00000000000 --- a/packages/syft/src/syft/node/node.py +++ /dev/null @@ -1,1771 +0,0 @@ -# future -from __future__ import annotations - -# stdlib -from collections import OrderedDict -from collections.abc import Callable -from datetime import MINYEAR -from datetime import datetime -from functools import partial -import hashlib -import json -import logging -import os -from pathlib import Path -import subprocess # nosec -import sys -from time import sleep -import traceback -from typing import Any -from typing import cast - -# third party -from nacl.signing import SigningKey -from result import Err -from result import Result - -# relative -from .. import __version__ -from ..abstract_node import AbstractNode -from ..abstract_node import NodeSideType -from ..abstract_node import NodeType -from ..client.api import SignedSyftAPICall -from ..client.api import SyftAPI -from ..client.api import SyftAPICall -from ..client.api import SyftAPIData -from ..client.api import debox_signed_syftapicall_response -from ..client.client import SyftClient -from ..exceptions.exception import PySyftException -from ..protocol.data_protocol import PROTOCOL_TYPE -from ..protocol.data_protocol import get_data_protocol -from ..service.action.action_object import Action -from ..service.action.action_object import ActionObject -from ..service.action.action_store import ActionStore -from ..service.action.action_store import DictActionStore -from ..service.action.action_store import MongoActionStore -from ..service.action.action_store import SQLiteActionStore -from ..service.blob_storage.service import BlobStorageService -from ..service.code.user_code_service import UserCodeService -from ..service.code.user_code_stash import UserCodeStash -from ..service.context import AuthedServiceContext -from ..service.context import NodeServiceContext -from ..service.context import UnauthedServiceContext -from ..service.context import UserLoginCredentials -from ..service.job.job_stash import Job -from ..service.job.job_stash import JobStash -from ..service.job.job_stash import JobStatus -from ..service.job.job_stash import JobType -from ..service.metadata.node_metadata import NodeMetadata -from ..service.network.network_service import NetworkService -from ..service.network.utils import PeerHealthCheckTask -from ..service.notifier.notifier_service import NotifierService -from ..service.queue.base_queue import AbstractMessageHandler -from ..service.queue.base_queue import QueueConsumer -from ..service.queue.base_queue import QueueProducer -from ..service.queue.queue import APICallMessageHandler -from ..service.queue.queue import QueueManager -from ..service.queue.queue_stash import APIEndpointQueueItem -from ..service.queue.queue_stash import ActionQueueItem -from ..service.queue.queue_stash import QueueItem -from ..service.queue.queue_stash import QueueStash -from ..service.queue.zmq_queue import QueueConfig -from ..service.queue.zmq_queue import ZMQClientConfig -from ..service.queue.zmq_queue import ZMQQueueConfig -from ..service.response import SyftError -from ..service.service import AbstractService -from ..service.service import ServiceConfigRegistry -from ..service.service import UserServiceConfigRegistry -from ..service.settings.settings import NodeSettings -from ..service.settings.settings import NodeSettingsUpdate -from ..service.settings.settings_stash import SettingsStash -from ..service.user.user import User -from ..service.user.user import UserCreate -from ..service.user.user import UserView -from ..service.user.user_roles import ServiceRole -from ..service.user.user_service import UserService -from ..service.user.user_stash import UserStash -from ..service.worker.utils import DEFAULT_WORKER_IMAGE_TAG -from ..service.worker.utils import DEFAULT_WORKER_POOL_NAME -from ..service.worker.utils import create_default_image -from ..service.worker.worker_image_service import SyftWorkerImageService -from ..service.worker.worker_pool import WorkerPool -from ..service.worker.worker_pool_service import SyftWorkerPoolService -from ..service.worker.worker_pool_stash import SyftWorkerPoolStash -from ..service.worker.worker_stash import WorkerStash -from ..store.blob_storage import BlobStorageConfig -from ..store.blob_storage.on_disk import OnDiskBlobStorageClientConfig -from ..store.blob_storage.on_disk import OnDiskBlobStorageConfig -from ..store.blob_storage.seaweedfs import SeaweedFSBlobDeposit -from ..store.dict_document_store import DictStoreConfig -from ..store.document_store import StoreConfig -from ..store.linked_obj import LinkedObject -from ..store.mongo_document_store import MongoStoreConfig -from ..store.sqlite_document_store import SQLiteStoreClientConfig -from ..store.sqlite_document_store import SQLiteStoreConfig -from ..types.datetime import DATETIME_FORMAT -from ..types.syft_metaclass import Empty -from ..types.syft_object import Context -from ..types.syft_object import PartialSyftObject -from ..types.syft_object import SYFT_OBJECT_VERSION_2 -from ..types.syft_object import SyftObject -from ..types.uid import UID -from ..util.experimental_flags import flags -from ..util.telemetry import instrument -from ..util.util import get_dev_mode -from ..util.util import get_env -from ..util.util import get_queue_address -from ..util.util import random_name -from ..util.util import str_to_bool -from ..util.util import thread_ident -from .credentials import SyftSigningKey -from .credentials import SyftVerifyKey -from .service_registry import ServiceRegistry -from .utils import get_named_node_uid -from .utils import get_temp_dir_for_node -from .utils import remove_temp_dir_for_node -from .worker_settings import WorkerSettings - -logger = logging.getLogger(__name__) - -# if user code needs to be serded and its not available we can call this to refresh -# the code for a specific node UID and thread -CODE_RELOADER: dict[int, Callable] = {} - - -NODE_PRIVATE_KEY = "NODE_PRIVATE_KEY" -NODE_UID = "NODE_UID" -NODE_TYPE = "NODE_TYPE" -NODE_NAME = "NODE_NAME" -NODE_SIDE_TYPE = "NODE_SIDE_TYPE" - -DEFAULT_ROOT_EMAIL = "DEFAULT_ROOT_EMAIL" -DEFAULT_ROOT_USERNAME = "DEFAULT_ROOT_USERNAME" -DEFAULT_ROOT_PASSWORD = "DEFAULT_ROOT_PASSWORD" # nosec - - -def get_private_key_env() -> str | None: - return get_env(NODE_PRIVATE_KEY) - - -def get_node_type() -> str | None: - return get_env(NODE_TYPE, "domain") - - -def get_node_name() -> str | None: - return get_env(NODE_NAME, None) - - -def get_node_side_type() -> str | None: - return get_env(NODE_SIDE_TYPE, "high") - - -def get_node_uid_env() -> str | None: - return get_env(NODE_UID) - - -def get_default_root_email() -> str | None: - return get_env(DEFAULT_ROOT_EMAIL, "info@openmined.org") - - -def get_default_root_username() -> str | None: - return get_env(DEFAULT_ROOT_USERNAME, "Jane Doe") - - -def get_default_root_password() -> str | None: - return get_env(DEFAULT_ROOT_PASSWORD, "changethis") # nosec - - -def get_enable_warnings() -> bool: - return str_to_bool(get_env("ENABLE_WARNINGS", "False")) - - -def get_container_host() -> str | None: - return get_env("CONTAINER_HOST") - - -def get_default_worker_image() -> str | None: - return get_env("DEFAULT_WORKER_POOL_IMAGE") - - -def get_default_worker_pool_name() -> str | None: - return get_env("DEFAULT_WORKER_POOL_NAME", DEFAULT_WORKER_POOL_NAME) - - -def get_default_bucket_name() -> str: - env = get_env("DEFAULT_BUCKET_NAME") - node_id = get_node_uid_env() or "syft-bucket" - return env or node_id or "syft-bucket" - - -def get_default_worker_pool_count(node: Node) -> int: - return int( - get_env( - "DEFAULT_WORKER_POOL_COUNT", node.queue_config.client_config.n_consumers - ) - ) - - -def get_default_worker_pool_pod_annotations() -> dict[str, str] | None: - annotations = get_env("DEFAULT_WORKER_POOL_POD_ANNOTATIONS", "null") - return json.loads(annotations) - - -def get_default_worker_pool_pod_labels() -> dict[str, str] | None: - labels = get_env("DEFAULT_WORKER_POOL_POD_LABELS", "null") - return json.loads(labels) - - -def in_kubernetes() -> bool: - return get_container_host() == "k8s" - - -def get_venv_packages() -> str: - try: - # subprocess call is safe because it uses a fully qualified path and fixed arguments - result = subprocess.run( - [sys.executable, "-m", "pip", "list", "--format=freeze"], # nosec - capture_output=True, - check=True, - text=True, - ) - return result.stdout - except subprocess.CalledProcessError as e: - return f"An error occurred: {e.stderr}" - - -def get_syft_worker() -> bool: - return str_to_bool(get_env("SYFT_WORKER", "false")) - - -def get_k8s_pod_name() -> str | None: - return get_env("K8S_POD_NAME") - - -def get_syft_worker_uid() -> str | None: - is_worker = get_syft_worker() - pod_name = get_k8s_pod_name() - uid = get_env("SYFT_WORKER_UID") - # if uid is empty is a K8S worker, generate a uid from the pod name - if (not uid) and is_worker and pod_name: - uid = str(UID.with_seed(pod_name)) - return uid - - -signing_key_env = get_private_key_env() -node_uid_env = get_node_uid_env() - -default_root_email = get_default_root_email() -default_root_username = get_default_root_username() -default_root_password = get_default_root_password() - - -class AuthNodeContextRegistry: - __node_context_registry__: dict[str, NodeServiceContext] = OrderedDict() - - @classmethod - def set_node_context( - cls, - node_uid: UID | str, - context: NodeServiceContext, - user_verify_key: SyftVerifyKey | str, - ) -> None: - if isinstance(node_uid, str): - node_uid = UID.from_string(node_uid) - - if isinstance(user_verify_key, str): - user_verify_key = SyftVerifyKey.from_string(user_verify_key) - - key = cls._get_key(node_uid=node_uid, user_verify_key=user_verify_key) - - cls.__node_context_registry__[key] = context - - @staticmethod - def _get_key(node_uid: UID, user_verify_key: SyftVerifyKey) -> str: - return "-".join(str(x) for x in (node_uid, user_verify_key)) - - @classmethod - def auth_context_for_user( - cls, - node_uid: UID, - user_verify_key: SyftVerifyKey, - ) -> AuthedServiceContext | None: - key = cls._get_key(node_uid=node_uid, user_verify_key=user_verify_key) - return cls.__node_context_registry__.get(key) - - -@instrument -class Node(AbstractNode): - signing_key: SyftSigningKey | None - required_signed_calls: bool = True - packages: str - - def __init__( - self, - *, # Trasterisk - name: str | None = None, - id: UID | None = None, - signing_key: SyftSigningKey | SigningKey | None = None, - action_store_config: StoreConfig | None = None, - document_store_config: StoreConfig | None = None, - root_email: str | None = default_root_email, - root_username: str | None = default_root_username, - root_password: str | None = default_root_password, - processes: int = 0, - is_subprocess: bool = False, - node_type: str | NodeType = NodeType.DOMAIN, - local_db: bool = False, - reset: bool = False, - blob_storage_config: BlobStorageConfig | None = None, - queue_config: QueueConfig | None = None, - queue_port: int | None = None, - n_consumers: int = 0, - create_producer: bool = False, - thread_workers: bool = False, - node_side_type: str | NodeSideType = NodeSideType.HIGH_SIDE, - enable_warnings: bool = False, - dev_mode: bool = False, - migrate: bool = False, - in_memory_workers: bool = True, - smtp_username: str | None = None, - smtp_password: str | None = None, - email_sender: str | None = None, - smtp_port: int | None = None, - smtp_host: str | None = None, - association_request_auto_approval: bool = False, - background_tasks: bool = False, - ): - # 🟡 TODO 22: change our ENV variable format and default init args to make this - # less horrible or add some convenience functions - self.dev_mode = dev_mode or get_dev_mode() - self.id = UID.from_string(node_uid_env) if node_uid_env else (id or UID()) - self.packages = "" - self.processes = processes - self.is_subprocess = is_subprocess - self.name = name or random_name() - self.enable_warnings = enable_warnings - self.in_memory_workers = in_memory_workers - self.node_type = NodeType(node_type) - self.node_side_type = NodeSideType(node_side_type) - self.client_cache: dict = {} - self.peer_client_cache: dict = {} - - if isinstance(node_type, str): - node_type = NodeType(node_type) - self.node_type = node_type - - if isinstance(node_side_type, str): - node_side_type = NodeSideType(node_side_type) - self.node_side_type = node_side_type - - skey = None - if signing_key_env: - skey = SyftSigningKey.from_string(signing_key_env) - elif isinstance(signing_key, SigningKey): - skey = SyftSigningKey(signing_key=signing_key) - else: - skey = signing_key - self.signing_key = skey or SyftSigningKey.generate() - - self.association_request_auto_approval = association_request_auto_approval - - self.queue_config = self.create_queue_config( - n_consumers=n_consumers, - create_producer=create_producer, - thread_workers=thread_workers, - queue_port=queue_port, - queue_config=queue_config, - ) - - # must call before initializing stores - if reset: - self.remove_temp_dir() - - use_sqlite = local_db or (processes > 0 and not is_subprocess) - document_store_config = document_store_config or self.get_default_store( - use_sqlite=use_sqlite, - store_type="Document Store", - ) - action_store_config = action_store_config or self.get_default_store( - use_sqlite=use_sqlite, - store_type="Action Store", - ) - self.init_stores( - action_store_config=action_store_config, - document_store_config=document_store_config, - ) - - # construct services only after init stores - self.services: ServiceRegistry = ServiceRegistry.for_node(self) - - create_admin_new( # nosec B106 - name=root_username, - email=root_email, - password=root_password, - node=self, - ) - - NotifierService.init_notifier( - node=self, - email_password=smtp_password, - email_username=smtp_username, - email_sender=email_sender, - smtp_port=smtp_port, - smtp_host=smtp_host, - ) - - self.post_init() - - if migrate: - self.find_and_migrate_data() - else: - self.find_and_migrate_data([NodeSettings]) - - self.create_initial_settings(admin_email=root_email) - - self.init_blob_storage(config=blob_storage_config) - - # Migrate data before any operation on db - - # first migrate, for backwards compatibility - self.init_queue_manager(queue_config=self.queue_config) - - context = AuthedServiceContext( - node=self, - credentials=self.verify_key, - role=ServiceRole.ADMIN, - ) - - self.peer_health_manager: PeerHealthCheckTask | None = None - if background_tasks: - self.run_peer_health_checks(context=context) - - NodeRegistry.set_node_for(self.id, self) - - @property - def runs_in_docker(self) -> bool: - path = "/proc/self/cgroup" - return ( - os.path.exists("/.dockerenv") - or os.path.isfile(path) - and any("docker" in line for line in open(path)) - ) - - def get_default_store(self, use_sqlite: bool, store_type: str) -> StoreConfig: - if use_sqlite: - path = self.get_temp_dir("db") - file_name: str = f"{self.id}.sqlite" - if self.dev_mode: - logger.debug(f"{store_type}'s SQLite DB path: {path/file_name}") - return SQLiteStoreConfig( - client_config=SQLiteStoreClientConfig( - filename=file_name, - path=path, - ) - ) - return DictStoreConfig() - - def init_blob_storage(self, config: BlobStorageConfig | None = None) -> None: - if config is None: - client_config = OnDiskBlobStorageClientConfig( - base_directory=self.get_temp_dir("blob") - ) - config_ = OnDiskBlobStorageConfig( - client_config=client_config, - min_blob_size=int(os.getenv("MIN_SIZE_BLOB_STORAGE_MB", "16")), - ) - else: - config_ = config - self.blob_store_config = config_ - self.blob_storage_client = config_.client_type(config=config_.client_config) - - # relative - from ..store.blob_storage.seaweedfs import SeaweedFSConfig - - if isinstance(config, SeaweedFSConfig) and self.signing_key: - blob_storage_service = self.get_service(BlobStorageService) - remote_profiles = blob_storage_service.remote_profile_stash.get_all( - credentials=self.signing_key.verify_key, has_permission=True - ).ok() - for remote_profile in remote_profiles: - self.blob_store_config.client_config.remote_profiles[ - remote_profile.profile_name - ] = remote_profile - - if self.dev_mode: - if isinstance(self.blob_store_config, OnDiskBlobStorageConfig): - logger.debug( - f"Using on-disk blob storage with path: " - f"{self.blob_store_config.client_config.base_directory}", - ) - logger.debug( - f"Minimum object size to be saved to the blob storage: " - f"{self.blob_store_config.min_blob_size} (MB)." - ) - - def run_peer_health_checks(self, context: AuthedServiceContext) -> None: - self.peer_health_manager = PeerHealthCheckTask() - self.peer_health_manager.run(context=context) - - def stop(self) -> None: - if self.peer_health_manager is not None: - self.peer_health_manager.stop() - - for consumer_list in self.queue_manager.consumers.values(): - for c in consumer_list: - c.close() - for p in self.queue_manager.producers.values(): - p.close() - - self.queue_manager.producers.clear() - self.queue_manager.consumers.clear() - - NodeRegistry.remove_node(self.id) - - def close(self) -> None: - self.stop() - - def cleanup(self) -> None: - self.stop() - self.remove_temp_dir() - - def create_queue_config( - self, - n_consumers: int, - create_producer: bool, - thread_workers: bool, - queue_port: int | None, - queue_config: QueueConfig | None, - ) -> QueueConfig: - if queue_config: - queue_config_ = queue_config - elif queue_port is not None or n_consumers > 0 or create_producer: - if not create_producer and queue_port is None: - logger.warn("No queue port defined to bind consumers.") - queue_config_ = ZMQQueueConfig( - client_config=ZMQClientConfig( - create_producer=create_producer, - queue_port=queue_port, - n_consumers=n_consumers, - ), - thread_workers=thread_workers, - ) - else: - queue_config_ = ZMQQueueConfig() - - return queue_config_ - - def init_queue_manager(self, queue_config: QueueConfig) -> None: - MessageHandlers = [APICallMessageHandler] - if self.is_subprocess: - return None - - self.queue_manager = QueueManager(config=queue_config) - for message_handler in MessageHandlers: - queue_name = message_handler.queue_name - # client config - if getattr(queue_config.client_config, "create_producer", True): - context = AuthedServiceContext( - node=self, - credentials=self.verify_key, - role=ServiceRole.ADMIN, - ) - producer: QueueProducer = self.queue_manager.create_producer( - queue_name=queue_name, - queue_stash=self.queue_stash, - context=context, - worker_stash=self.worker_stash, - ) - producer.run() - address = producer.address - else: - port = queue_config.client_config.queue_port - if port is not None: - address = get_queue_address(port) - else: - address = None - - if address is None and queue_config.client_config.n_consumers > 0: - raise ValueError("address unknown for consumers") - - service_name = queue_config.client_config.consumer_service - - if not service_name: - # Create consumers for default worker pool - create_default_worker_pool(self) - else: - # Create consumer for given worker pool - syft_worker_uid = get_syft_worker_uid() - logger.info( - f"Running as consumer with uid={syft_worker_uid} service={service_name}" - ) - - if syft_worker_uid: - self.add_consumer_for_service( - service_name=service_name, - syft_worker_id=UID(syft_worker_uid), - address=address, - message_handler=message_handler, - ) - - def add_consumer_for_service( - self, - service_name: str, - syft_worker_id: UID, - address: str, - message_handler: type[AbstractMessageHandler] = APICallMessageHandler, - ) -> None: - consumer: QueueConsumer = self.queue_manager.create_consumer( - message_handler, - address=address, - service_name=service_name, - worker_stash=self.worker_stash, - syft_worker_id=syft_worker_id, - ) - consumer.run() - - def remove_consumer_with_id(self, syft_worker_id: UID) -> None: - for consumers in self.queue_manager.consumers.values(): - # Grab the list of consumers for the given queue - consumer_to_pop = None - for consumer_idx, consumer in enumerate(consumers): - if consumer.syft_worker_id == syft_worker_id: - consumer.close() - consumer_to_pop = consumer_idx - break - if consumer_to_pop is not None: - consumers.pop(consumer_to_pop) - - @classmethod - def named( - cls: type[Node], - *, # Trasterisk - name: str, - processes: int = 0, - reset: bool = False, - local_db: bool = False, - node_type: str | NodeType = NodeType.DOMAIN, - node_side_type: str | NodeSideType = NodeSideType.HIGH_SIDE, - enable_warnings: bool = False, - n_consumers: int = 0, - thread_workers: bool = False, - create_producer: bool = False, - queue_port: int | None = None, - dev_mode: bool = False, - migrate: bool = False, - in_memory_workers: bool = True, - association_request_auto_approval: bool = False, - background_tasks: bool = False, - ) -> Node: - uid = get_named_node_uid(name) - name_hash = hashlib.sha256(name.encode("utf8")).digest() - key = SyftSigningKey(signing_key=SigningKey(name_hash)) - blob_storage_config = None - - node_type = NodeType(node_type) - node_side_type = NodeSideType(node_side_type) - - return cls( - name=name, - id=uid, - signing_key=key, - processes=processes, - local_db=local_db, - node_type=node_type, - node_side_type=node_side_type, - enable_warnings=enable_warnings, - blob_storage_config=blob_storage_config, - queue_port=queue_port, - n_consumers=n_consumers, - thread_workers=thread_workers, - create_producer=create_producer, - dev_mode=dev_mode, - migrate=migrate, - in_memory_workers=in_memory_workers, - reset=reset, - association_request_auto_approval=association_request_auto_approval, - background_tasks=background_tasks, - ) - - def is_root(self, credentials: SyftVerifyKey) -> bool: - return credentials == self.verify_key - - @property - def root_client(self) -> SyftClient: - # relative - from ..client.client import PythonConnection - - connection = PythonConnection(node=self) - client_type = connection.get_client_type() - if isinstance(client_type, SyftError): - return client_type - root_client = client_type(connection=connection, credentials=self.signing_key) - if root_client.api.refresh_api_callback is not None: - root_client.api.refresh_api_callback() - return root_client - - def _find_klasses_pending_for_migration( - self, object_types: list[SyftObject] - ) -> list[SyftObject]: - context = AuthedServiceContext( - node=self, - credentials=self.verify_key, - role=ServiceRole.ADMIN, - ) - migration_state_service = self.services.migration - - klasses_to_be_migrated = [] - - for object_type in object_types: - canonical_name = object_type.__canonical_name__ - object_version = object_type.__version__ - - migration_state = migration_state_service.get_state(context, canonical_name) - if isinstance(migration_state, SyftError): - raise Exception( - f"Failed to get migration state for {canonical_name}. Error: {migration_state}" - ) - if ( - migration_state is not None - and migration_state.current_version != migration_state.latest_version - ): - klasses_to_be_migrated.append(object_type) - else: - migration_state_service.register_migration_state( - context, - current_version=object_version, - canonical_name=canonical_name, - ) - - return klasses_to_be_migrated - - def find_and_migrate_data( - self, document_store_object_types: list[type[SyftObject]] | None = None - ) -> None: - context = AuthedServiceContext( - node=self, - credentials=self.verify_key, - role=ServiceRole.ADMIN, - ) - migration_service = self.get_service("migrationservice") - return migration_service.migrate_data(context, document_store_object_types) - - @property - def guest_client(self) -> SyftClient: - return self.get_guest_client() - - @property - def current_protocol(self) -> str | int: - data_protocol = get_data_protocol() - return data_protocol.latest_version - - def get_guest_client(self, verbose: bool = True) -> SyftClient: - # relative - from ..client.client import PythonConnection - - connection = PythonConnection(node=self) - if verbose and self.node_side_type: - message: str = ( - f"Logged into <{self.name}: {self.node_side_type.value.capitalize()} " - ) - if self.node_type: - message += f"side {self.node_type.value.capitalize()} > as GUEST" - logger.debug(message) - - client_type = connection.get_client_type() - if isinstance(client_type, SyftError): - return client_type - - guest_client = client_type( - connection=connection, credentials=SyftSigningKey.generate() - ) - if guest_client.api.refresh_api_callback is not None: - guest_client.api.refresh_api_callback() - return guest_client - - def __repr__(self) -> str: - service_string = "" - if not self.is_subprocess: - services = [service.__name__ for service in self.services] - service_string = ", ".join(sorted(services)) - service_string = f"\n\nServices:\n{service_string}" - return f"{type(self).__name__}: {self.name} - {self.id} - {self.node_type}{service_string}" - - def post_init(self) -> None: - context = AuthedServiceContext( - node=self, credentials=self.verify_key, role=ServiceRole.ADMIN - ) - AuthNodeContextRegistry.set_node_context( - node_uid=self.id, user_verify_key=self.verify_key, context=context - ) - - if "usercodeservice" in self.service_path_map: - user_code_service = self.get_service(UserCodeService) - user_code_service.load_user_code(context=context) - - def reload_user_code() -> None: - user_code_service.load_user_code(context=context) - - ti = thread_ident() - if ti is not None: - CODE_RELOADER[ti] = reload_user_code - - def init_stores( - self, - document_store_config: StoreConfig, - action_store_config: StoreConfig, - ) -> None: - # We add the python id of the current node in order - # to create one connection per Node object in MongoClientCache - # so that we avoid closing the connection from a - # different thread through the garbage collection - if isinstance(document_store_config, MongoStoreConfig): - document_store_config.client_config.node_obj_python_id = id(self) - - self.document_store_config = document_store_config - self.document_store = document_store_config.store_type( - node_uid=self.id, - root_verify_key=self.verify_key, - store_config=document_store_config, - ) - - if isinstance(action_store_config, SQLiteStoreConfig): - self.action_store: ActionStore = SQLiteActionStore( - node_uid=self.id, - store_config=action_store_config, - root_verify_key=self.verify_key, - document_store=self.document_store, - ) - elif isinstance(action_store_config, MongoStoreConfig): - # We add the python id of the current node in order - # to create one connection per Node object in MongoClientCache - # so that we avoid closing the connection from a - # different thread through the garbage collection - action_store_config.client_config.node_obj_python_id = id(self) - - self.action_store = MongoActionStore( - node_uid=self.id, - root_verify_key=self.verify_key, - store_config=action_store_config, - document_store=self.document_store, - ) - else: - self.action_store = DictActionStore( - node_uid=self.id, - root_verify_key=self.verify_key, - document_store=self.document_store, - ) - - self.action_store_config = action_store_config - self.queue_stash = QueueStash(store=self.document_store) - - @property - def job_stash(self) -> JobStash: - return self.get_service("jobservice").stash - - @property - def worker_stash(self) -> WorkerStash: - return self.get_service("workerservice").stash - - @property - def service_path_map(self) -> dict[str, AbstractService]: - return self.services.service_path_map - - @property - def initialized_services(self) -> list[AbstractService]: - return self.services.services - - def get_service_method(self, path_or_func: str | Callable) -> Callable: - if callable(path_or_func): - path_or_func = path_or_func.__qualname__ - return self._get_service_method_from_path(path_or_func) - - def get_service(self, path_or_func: str | Callable) -> AbstractService: - return self.services.get_service(path_or_func) - - def _get_service_method_from_path(self, path: str) -> Callable: - path_list = path.split(".") - method_name = path_list.pop() - service_obj = self.services._get_service_from_path(path=path) - - return getattr(service_obj, method_name) - - def get_temp_dir(self, dir_name: str = "") -> Path: - """ - Get a temporary directory unique to the node. - Provide all dbs, blob dirs, and locks using this directory. - """ - return get_temp_dir_for_node(self.id, dir_name) - - def remove_temp_dir(self) -> None: - """ - Remove the temporary directory for this node. - """ - remove_temp_dir_for_node(self.id) - - def update_self(self, settings: NodeSettings) -> None: - updateable_attrs = ( - NodeSettingsUpdate.model_fields.keys() - - PartialSyftObject.model_fields.keys() - ) - for attr_name in updateable_attrs: - attr = getattr(settings, attr_name) - if attr is not Empty: - setattr(self, attr_name, attr) - - @property - def settings(self) -> NodeSettings: - settings_stash = SettingsStash(store=self.document_store) - if self.signing_key is None: - raise ValueError(f"{self} has no signing key") - settings = settings_stash.get_all(self.signing_key.verify_key) - if settings.is_err(): - raise ValueError( - f"Cannot get node settings for '{self.name}'. Error: {settings.err()}" - ) - if settings.is_ok() and len(settings.ok()) > 0: - settings = settings.ok()[0] - self.update_self(settings) - return settings - - @property - def metadata(self) -> NodeMetadata: - show_warnings = self.enable_warnings - settings_data = self.settings - name = settings_data.name - organization = settings_data.organization - description = settings_data.description - show_warnings = settings_data.show_warnings - node_type = settings_data.node_type.value if settings_data.node_type else "" - node_side_type = ( - settings_data.node_side_type.value if settings_data.node_side_type else "" - ) - eager_execution_enabled = settings_data.eager_execution_enabled - - return NodeMetadata( - name=name, - id=self.id, - verify_key=self.verify_key, - highest_version=SYFT_OBJECT_VERSION_2, - lowest_version=SYFT_OBJECT_VERSION_2, - syft_version=__version__, - description=description, - organization=organization, - node_type=node_type, - node_side_type=node_side_type, - show_warnings=show_warnings, - eager_execution_enabled=eager_execution_enabled, - min_size_blob_storage_mb=self.blob_store_config.min_blob_size, - ) - - @property - def icon(self) -> str: - return "🦾" - - @property - def verify_key(self) -> SyftVerifyKey: - if self.signing_key is None: - raise ValueError(f"{self} has no signing key") - return self.signing_key.verify_key - - def __hash__(self) -> int: - return hash(self.id) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, type(self)): - return False - - if self.id != other.id: - return False - - return True - - def await_future( - self, credentials: SyftVerifyKey, uid: UID - ) -> QueueItem | None | SyftError: - # stdlib - - # relative - from ..service.queue.queue import Status - - while True: - result = self.queue_stash.pop_on_complete(credentials, uid) - if not result.is_ok(): - return result.err() - else: - res = result.ok() - if res.status == Status.COMPLETED: - return res - sleep(0.1) - - def resolve_future( - self, credentials: SyftVerifyKey, uid: UID - ) -> QueueItem | None | SyftError: - result = self.queue_stash.pop_on_complete(credentials, uid) - - if result.is_ok(): - queue_obj = result.ok() - queue_obj._set_obj_location_( - node_uid=self.id, - credentials=credentials, - ) - return queue_obj - return result.err() - - def forward_message( - self, api_call: SyftAPICall | SignedSyftAPICall - ) -> Result | QueueItem | SyftObject | SyftError | Any: - node_uid = api_call.message.node_uid - if "networkservice" not in self.service_path_map: - return SyftError( - message=( - "Node has no network service so we can't " - f"forward this message to {node_uid}" - ) - ) - - client = None - - network_service = self.get_service(NetworkService) - peer = network_service.stash.get_by_uid(self.verify_key, node_uid) - - if peer.is_ok() and peer.ok(): - peer = peer.ok() - - # Since we have several routes to a peer - # we need to cache the client for a given node_uid along with the route - peer_cache_key = hash(node_uid) + hash(peer.pick_highest_priority_route()) - if peer_cache_key in self.peer_client_cache: - client = self.peer_client_cache[peer_cache_key] - else: - context = AuthedServiceContext( - node=self, credentials=api_call.credentials - ) - - client = peer.client_with_context(context=context) - if client.is_err(): - return SyftError( - message=f"Failed to create remote client for peer: " - f"{peer.id}. Error: {client.err()}" - ) - client = client.ok() - - self.peer_client_cache[peer_cache_key] = client - - if client: - message: SyftAPICall = api_call.message - if message.path == "metadata": - result = client.metadata - elif message.path == "login": - result = client.connection.login(**message.kwargs) - elif message.path == "register": - result = client.connection.register(**message.kwargs) - elif message.path == "api": - result = client.connection.get_api(**message.kwargs) - else: - signed_result = client.connection.make_call(api_call) - result = debox_signed_syftapicall_response(signed_result=signed_result) - - # relative - from ..store.blob_storage import BlobRetrievalByURL - - if isinstance(result, BlobRetrievalByURL | SeaweedFSBlobDeposit): - result.proxy_node_uid = peer.id - - return result - - return SyftError(message=(f"Node has no route to {node_uid}")) - - def get_role_for_credentials(self, credentials: SyftVerifyKey) -> ServiceRole: - role = self.get_service("userservice").get_role_for_credentials( - credentials=credentials - ) - return role - - def handle_api_call( - self, - api_call: SyftAPICall | SignedSyftAPICall, - job_id: UID | None = None, - check_call_location: bool = True, - ) -> Result[SignedSyftAPICall, Err]: - # Get the result - result = self.handle_api_call_with_unsigned_result( - api_call, job_id=job_id, check_call_location=check_call_location - ) - # Sign the result - signed_result = SyftAPIData(data=result).sign(self.signing_key) - - return signed_result - - def handle_api_call_with_unsigned_result( - self, - api_call: SyftAPICall | SignedSyftAPICall, - job_id: UID | None = None, - check_call_location: bool = True, - ) -> Result | QueueItem | SyftObject | SyftError: - if self.required_signed_calls and isinstance(api_call, SyftAPICall): - return SyftError( - message=f"You sent a {type(api_call)}. This node requires SignedSyftAPICall." - ) - elif not api_call.is_valid: - return SyftError(message="Your message signature is invalid") - - if api_call.message.node_uid != self.id and check_call_location: - return self.forward_message(api_call=api_call) - - if api_call.message.path == "queue": - return self.resolve_future( - credentials=api_call.credentials, uid=api_call.message.kwargs["uid"] - ) - - if api_call.message.path == "metadata": - return self.metadata - - result = None - is_blocking = api_call.message.blocking - - if is_blocking or self.is_subprocess: - credentials: SyftVerifyKey = api_call.credentials - api_call = api_call.message - - role = self.get_role_for_credentials(credentials=credentials) - context = AuthedServiceContext( - node=self, - credentials=credentials, - role=role, - job_id=job_id, - is_blocking_api_call=is_blocking, - ) - AuthNodeContextRegistry.set_node_context(self.id, context, credentials) - - user_config_registry = UserServiceConfigRegistry.from_role(role) - - if api_call.path not in user_config_registry: - if ServiceConfigRegistry.path_exists(api_call.path): - return SyftError( - message=f"As a `{role}`, " - f"you have no access to: {api_call.path}" - ) - else: - return SyftError( - message=f"API call not in registered services: {api_call.path}" - ) - - _private_api_path = user_config_registry.private_path_for(api_call.path) - method = self.get_service_method(_private_api_path) - try: - logger.info(f"API Call: {api_call}") - result = method(context, *api_call.args, **api_call.kwargs) - except PySyftException as e: - return e.handle() - except Exception: - result = SyftError( - message=f"Exception calling {api_call.path}. {traceback.format_exc()}" - ) - else: - return self.add_api_call_to_queue(api_call) - return result - - def add_api_endpoint_execution_to_queue( - self, - credentials: SyftVerifyKey, - method: str, - path: str, - *args: Any, - worker_pool: str | None = None, - **kwargs: Any, - ) -> Job | SyftError: - job_id = UID() - task_uid = UID() - worker_settings = WorkerSettings.from_node(node=self) - - if worker_pool is None: - worker_pool = self.get_default_worker_pool() - else: - worker_pool = self.get_worker_pool_by_name(worker_pool) - - if isinstance(worker_pool, SyftError): - return worker_pool - elif worker_pool is None: - return SyftError(message="Worker pool not found") - - # Create a Worker pool reference object - worker_pool_ref = LinkedObject.from_obj( - worker_pool, - service_type=SyftWorkerPoolService, - node_uid=self.id, - ) - queue_item = APIEndpointQueueItem( - id=task_uid, - method=method, - node_uid=self.id, - syft_client_verify_key=credentials, - syft_node_location=self.id, - job_id=job_id, - worker_settings=worker_settings, - args=args, - kwargs={"path": path, **kwargs}, - has_execute_permissions=True, - worker_pool=worker_pool_ref, # set worker pool reference as part of queue item - ) - - action = Action.from_api_endpoint_execution() - return self.add_queueitem_to_queue( - queue_item=queue_item, - credentials=credentials, - action=action, - job_type=JobType.TWINAPIJOB, - ) - - def get_worker_pool_ref_by_name( - self, credentials: SyftVerifyKey, worker_pool_name: str | None = None - ) -> LinkedObject | SyftError: - # If worker pool id is not set, then use default worker pool - # Else, get the worker pool for given uid - if worker_pool_name is None: - worker_pool = self.get_default_worker_pool() - else: - result = self.pool_stash.get_by_name(credentials, worker_pool_name) - if result.is_err(): - return SyftError(message=f"{result.err()}") - worker_pool = result.ok() - - # Create a Worker pool reference object - worker_pool_ref = LinkedObject.from_obj( - worker_pool, - service_type=SyftWorkerPoolService, - node_uid=self.id, - ) - return worker_pool_ref - - def add_action_to_queue( - self, - action: Action, - credentials: SyftVerifyKey, - parent_job_id: UID | None = None, - has_execute_permissions: bool = False, - worker_pool_name: str | None = None, - ) -> Job | SyftError: - job_id = UID() - task_uid = UID() - worker_settings = WorkerSettings.from_node(node=self) - - # Extract worker pool id from user code - if action.user_code_id is not None: - result = self.user_code_stash.get_by_uid( - credentials=credentials, uid=action.user_code_id - ) - - # If result is Ok, then user code object exists - if result.is_ok() and result.ok() is not None: - user_code = result.ok() - worker_pool_name = user_code.worker_pool_name - - worker_pool_ref = self.get_worker_pool_ref_by_name( - credentials, worker_pool_name - ) - if isinstance(worker_pool_ref, SyftError): - return worker_pool_ref - queue_item = ActionQueueItem( - id=task_uid, - node_uid=self.id, - syft_client_verify_key=credentials, - syft_node_location=self.id, - job_id=job_id, - worker_settings=worker_settings, - args=[], - kwargs={"action": action}, - has_execute_permissions=has_execute_permissions, - worker_pool=worker_pool_ref, # set worker pool reference as part of queue item - ) - user_id = self.get_service("UserService").get_user_id_for_credentials( - credentials - ) - - return self.add_queueitem_to_queue( - queue_item=queue_item, - credentials=credentials, - action=action, - parent_job_id=parent_job_id, - user_id=user_id, - ) - - def add_queueitem_to_queue( - self, - *, - queue_item: QueueItem, - credentials: SyftVerifyKey, - action: Action | None = None, - parent_job_id: UID | None = None, - user_id: UID | None = None, - job_type: JobType = JobType.JOB, - ) -> Job | SyftError: - log_id = UID() - role = self.get_role_for_credentials(credentials=credentials) - context = AuthedServiceContext(node=self, credentials=credentials, role=role) - - result_obj = ActionObject.empty() - if action is not None: - result_obj = ActionObject.obj_not_ready(id=action.result_id) - result_obj.id = action.result_id - result_obj.syft_resolved = False - result_obj.syft_node_location = self.id - result_obj.syft_client_verify_key = credentials - - action_service = self.get_service("actionservice") - - if not action_service.store.exists(uid=action.result_id): - result = action_service.set_result_to_store( - result_action_object=result_obj, - context=context, - ) - if result.is_err(): - return result.err() - - job = Job( - id=queue_item.job_id, - result=result_obj, - node_uid=self.id, - syft_client_verify_key=credentials, - syft_node_location=self.id, - log_id=log_id, - parent_job_id=parent_job_id, - action=action, - requested_by=user_id, - job_type=job_type, - ) - - # 🟡 TODO 36: Needs distributed lock - job_res = self.job_stash.set(credentials, job) - if job_res.is_err(): - return SyftError(message=f"{job_res.err()}") - self.queue_stash.set_placeholder(credentials, queue_item) - - log_service = self.get_service("logservice") - - result = log_service.add(context, log_id, queue_item.job_id) - if isinstance(result, SyftError): - return result - return job - - def _sort_jobs(self, jobs: list[Job]) -> list[Job]: - job_datetimes = {} - for job in jobs: - try: - d = datetime.strptime(job.creation_time, DATETIME_FORMAT) - except Exception: - d = datetime(MINYEAR, 1, 1) - job_datetimes[job.id] = d - - jobs.sort( - key=lambda job: (job.status != JobStatus.COMPLETED, job_datetimes[job.id]), - reverse=True, - ) - - return jobs - - def _get_existing_user_code_jobs( - self, context: AuthedServiceContext, user_code_id: UID - ) -> list[Job] | SyftError: - job_service = self.get_service("jobservice") - jobs = job_service.get_by_user_code_id( - context=context, user_code_id=user_code_id - ) - - if isinstance(jobs, SyftError): - return jobs - - return self._sort_jobs(jobs) - - def _is_usercode_call_on_owned_kwargs( - self, - context: AuthedServiceContext, - api_call: SyftAPICall, - user_code_id: UID, - ) -> bool: - if api_call.path != "code.call": - return False - user_code_service = self.get_service("usercodeservice") - return user_code_service.is_execution_on_owned_args( - context, user_code_id, api_call.kwargs - ) - - def add_api_call_to_queue( - self, api_call: SyftAPICall, parent_job_id: UID | None = None - ) -> Job | SyftError: - unsigned_call = api_call - if isinstance(api_call, SignedSyftAPICall): - unsigned_call = api_call.message - - credentials = api_call.credentials - context = AuthedServiceContext( - node=self, - credentials=credentials, - role=self.get_role_for_credentials(credentials=credentials), - ) - - is_user_code = unsigned_call.path == "code.call" - - service_str, method_str = unsigned_call.path.split(".") - - action = None - if is_user_code: - action = Action.from_api_call(unsigned_call) - user_code_id = action.user_code_id - - user = self.get_service(UserService).get_current_user(context) - if isinstance(user, SyftError): - return user - user = cast(UserView, user) - - is_execution_on_owned_kwargs_allowed = ( - user.mock_execution_permission or context.role == ServiceRole.ADMIN - ) - is_usercode_call_on_owned_kwargs = self._is_usercode_call_on_owned_kwargs( - context, unsigned_call, user_code_id - ) - # Low side does not execute jobs, unless this is a mock execution - if ( - not is_usercode_call_on_owned_kwargs - and self.node_side_type == NodeSideType.LOW_SIDE - ): - existing_jobs = self._get_existing_user_code_jobs(context, user_code_id) - if isinstance(existing_jobs, SyftError): - return existing_jobs - elif len(existing_jobs) > 0: - # Print warning if there are existing jobs for this user code - # relative - from ..util.util import prompt_warning_message - - prompt_warning_message( - "There are existing jobs for this user code, returning the latest one" - ) - return existing_jobs[-1] - else: - return SyftError( - message="Please wait for the admin to allow the execution of this code" - ) - - elif ( - is_usercode_call_on_owned_kwargs - and not is_execution_on_owned_kwargs_allowed - ): - return SyftError( - message="You do not have the permissions for mock execution, please contact the admin" - ) - - return self.add_action_to_queue( - action, api_call.credentials, parent_job_id=parent_job_id - ) - - else: - worker_settings = WorkerSettings.from_node(node=self) - worker_pool_ref = self.get_worker_pool_ref_by_name(credentials=credentials) - if isinstance(worker_pool_ref, SyftError): - return worker_pool_ref - - queue_item = QueueItem( - id=UID(), - node_uid=self.id, - syft_client_verify_key=api_call.credentials, - syft_node_location=self.id, - job_id=UID(), - worker_settings=worker_settings, - service=service_str, - method=method_str, - args=unsigned_call.args, - kwargs=unsigned_call.kwargs, - worker_pool=worker_pool_ref, - ) - return self.add_queueitem_to_queue( - queue_item=queue_item, - credentials=api_call.credentials, - action=None, - parent_job_id=parent_job_id, - ) - - @property - def pool_stash(self) -> SyftWorkerPoolStash: - return self.get_service(SyftWorkerPoolService).stash - - @property - def user_code_stash(self) -> UserCodeStash: - return self.get_service(UserCodeService).stash - - def get_default_worker_pool(self) -> WorkerPool | None | SyftError: - result = self.pool_stash.get_by_name( - credentials=self.verify_key, - pool_name=self.settings.default_worker_pool, - ) - if result.is_err(): - return SyftError(message=f"{result.err()}") - worker_pool = result.ok() - return worker_pool - - def get_worker_pool_by_name(self, name: str) -> WorkerPool | None | SyftError: - result = self.pool_stash.get_by_name( - credentials=self.verify_key, pool_name=name - ) - if result.is_err(): - return SyftError(message=f"{result.err()}") - worker_pool = result.ok() - return worker_pool - - def get_api( - self, - for_user: SyftVerifyKey | None = None, - communication_protocol: PROTOCOL_TYPE | None = None, - ) -> SyftAPI: - return SyftAPI.for_user( - node=self, - user_verify_key=for_user, - communication_protocol=communication_protocol, - ) - - def get_method_with_context( - self, function: Callable, context: NodeServiceContext - ) -> Callable: - method = self.get_service_method(function) - return partial(method, context=context) - - def get_unauthed_context( - self, login_credentials: UserLoginCredentials - ) -> NodeServiceContext: - return UnauthedServiceContext(node=self, login_credentials=login_credentials) - - def create_initial_settings(self, admin_email: str) -> NodeSettings | None: - try: - settings_stash = SettingsStash(store=self.document_store) - if self.signing_key is None: - logger.debug( - "create_initial_settings failed as there is no signing key" - ) - return None - settings_exists = settings_stash.get_all(self.signing_key.verify_key).ok() - if settings_exists: - node_settings = settings_exists[0] - if node_settings.__version__ != NodeSettings.__version__: - context = Context() - node_settings = node_settings.migrate_to( - NodeSettings.__version__, context - ) - res = settings_stash.delete_by_uid( - self.signing_key.verify_key, node_settings.id - ) - if res.is_err(): - raise Exception(res.value) - res = settings_stash.set(self.signing_key.verify_key, node_settings) - if res.is_err(): - raise Exception(res.value) - self.name = node_settings.name - self.association_request_auto_approval = ( - node_settings.association_request_auto_approval - ) - return None - else: - # Currently we allow automatic user registration on enclaves, - # as enclaves do not have superusers - if self.node_type == NodeType.ENCLAVE: - flags.CAN_REGISTER = True - new_settings = NodeSettings( - id=self.id, - name=self.name, - verify_key=self.verify_key, - node_type=self.node_type, - deployed_on=datetime.now().date().strftime("%m/%d/%Y"), - signup_enabled=flags.CAN_REGISTER, - admin_email=admin_email, - node_side_type=self.node_side_type.value, # type: ignore - show_warnings=self.enable_warnings, - association_request_auto_approval=self.association_request_auto_approval, - default_worker_pool=get_default_worker_pool_name(), - ) - result = settings_stash.set( - credentials=self.signing_key.verify_key, settings=new_settings - ) - if result.is_ok(): - return result.ok() - return None - except Exception as e: - logger.error("create_initial_settings failed", exc_info=e) - return None - - -def create_admin_new( - name: str, - email: str, - password: str, - node: AbstractNode, -) -> User | None: - try: - user_stash = UserStash(store=node.document_store) - row_exists = user_stash.get_by_email( - credentials=node.signing_key.verify_key, email=email - ).ok() - if row_exists: - return None - else: - create_user = UserCreate( - name=name, - email=email, - password=password, - password_verify=password, - role=ServiceRole.ADMIN, - ) - # New User Initialization - # 🟡 TODO: change later but for now this gives the main user super user automatically - user = create_user.to(User) - user.signing_key = node.signing_key - user.verify_key = user.signing_key.verify_key - result = user_stash.set( - credentials=node.signing_key.verify_key, - user=user, - ignore_duplicates=True, - ) - if result.is_ok(): - return result.ok() - else: - raise Exception(f"Could not create user: {result}") - except Exception as e: - logger.error("Unable to create new admin", exc_info=e) - - return None - - -class NodeRegistry: - __node_registry__: dict[UID, Node] = {} - - @classmethod - def set_node_for( - cls, - node_uid: UID | str, - node: Node, - ) -> None: - if isinstance(node_uid, str): - node_uid = UID.from_string(node_uid) - - cls.__node_registry__[node_uid] = node - - @classmethod - def node_for(cls, node_uid: UID) -> Node: - return cls.__node_registry__.get(node_uid, None) - - @classmethod - def get_all_nodes(cls) -> list[Node]: - return list(cls.__node_registry__.values()) - - @classmethod - def remove_node(cls, node_uid: UID) -> None: - if node_uid in cls.__node_registry__: - del cls.__node_registry__[node_uid] - - -def get_default_worker_tag_by_env(dev_mode: bool = False) -> str | None: - if in_kubernetes(): - return get_default_worker_image() - elif dev_mode: - return "local-dev" - else: - return __version__ - - -def create_default_worker_pool(node: Node) -> SyftError | None: - credentials = node.verify_key - pull_image = not node.dev_mode - image_stash = node.get_service(SyftWorkerImageService).stash - default_pool_name = node.settings.default_worker_pool - default_worker_pool = node.get_default_worker_pool() - default_worker_tag = get_default_worker_tag_by_env(node.dev_mode) - default_worker_pool_pod_annotations = get_default_worker_pool_pod_annotations() - default_worker_pool_pod_labels = get_default_worker_pool_pod_labels() - worker_count = get_default_worker_pool_count(node) - context = AuthedServiceContext( - node=node, - credentials=credentials, - role=ServiceRole.ADMIN, - ) - - if isinstance(default_worker_pool, SyftError): - logger.error( - f"Failed to get default worker pool {default_pool_name}. " - f"Error: {default_worker_pool.message}" - ) - return default_worker_pool - - logger.info(f"Creating default worker image with tag='{default_worker_tag}'. ") - # Get/Create a default worker SyftWorkerImage - default_image = create_default_image( - credentials=credentials, - image_stash=image_stash, - tag=default_worker_tag, - in_kubernetes=in_kubernetes(), - ) - if isinstance(default_image, SyftError): - logger.error(f"Failed to create default worker image: {default_image.message}") - return default_image - - if not default_image.is_built: - logger.info(f"Building default worker image with tag={default_worker_tag}. ") - image_build_method = node.get_service_method(SyftWorkerImageService.build) - # Build the Image for given tag - result = image_build_method( - context, - image_uid=default_image.id, - tag=DEFAULT_WORKER_IMAGE_TAG, - pull_image=pull_image, - ) - - if isinstance(result, SyftError): - logger.error(f"Failed to build default worker image: {result.message}") - return None - - # Create worker pool if it doesn't exists - logger.info( - "Setting up worker pool" - f"name={default_pool_name} " - f"workers={worker_count} " - f"image_uid={default_image.id} " - f"in_memory={node.in_memory_workers}. " - ) - if default_worker_pool is None: - worker_to_add_ = worker_count - create_pool_method = node.get_service_method(SyftWorkerPoolService.launch) - result = create_pool_method( - context, - pool_name=default_pool_name, - image_uid=default_image.id, - num_workers=worker_count, - pod_annotations=default_worker_pool_pod_annotations, - pod_labels=default_worker_pool_pod_labels, - ) - else: - # Else add a worker to existing worker pool - worker_to_add_ = max(default_worker_pool.max_count, worker_count) - len( - default_worker_pool.worker_list - ) - if worker_to_add_ > 0: - add_worker_method = node.get_service_method( - SyftWorkerPoolService.add_workers - ) - result = add_worker_method( - context=context, - number=worker_to_add_, - pool_name=default_pool_name, - ) - else: - return None - - if isinstance(result, SyftError): - logger.info(f"Default worker pool error. {result.message}") - return None - - for n in range(worker_to_add_): - container_status = result[n] - if container_status.error: - logger.error( - f"Failed to create container: Worker: {container_status.worker}," - f"Error: {container_status.error}" - ) - return None - - logger.info("Created default worker pool.") - return None diff --git a/packages/syft/src/syft/node/routes.py b/packages/syft/src/syft/node/routes.py deleted file mode 100644 index 37baaff90e8..00000000000 --- a/packages/syft/src/syft/node/routes.py +++ /dev/null @@ -1,273 +0,0 @@ -# stdlib -import base64 -import binascii -from collections.abc import AsyncGenerator -import logging -from typing import Annotated - -# third party -from fastapi import APIRouter -from fastapi import Body -from fastapi import Depends -from fastapi import HTTPException -from fastapi import Request -from fastapi import Response -from fastapi.responses import JSONResponse -from fastapi.responses import StreamingResponse -from pydantic import ValidationError -import requests - -# relative -from ..abstract_node import AbstractNode -from ..client.connection import NodeConnection -from ..protocol.data_protocol import PROTOCOL_TYPE -from ..serde.deserialize import _deserialize as deserialize -from ..serde.serialize import _serialize as serialize -from ..service.context import NodeServiceContext -from ..service.context import UnauthedServiceContext -from ..service.metadata.node_metadata import NodeMetadataJSON -from ..service.response import SyftError -from ..service.user.user import UserCreate -from ..service.user.user import UserPrivateKey -from ..service.user.user_service import UserService -from ..types.uid import UID -from ..util.telemetry import TRACE_MODE -from .credentials import SyftVerifyKey -from .credentials import UserLoginCredentials -from .worker import Worker - -logger = logging.getLogger(__name__) - - -def make_routes(worker: Worker) -> APIRouter: - if TRACE_MODE: - # third party - try: - # third party - from opentelemetry import trace - from opentelemetry.propagate import extract - except Exception as e: - logger.error("Failed to import opentelemetry", exc_info=e) - - router = APIRouter() - - async def get_body(request: Request) -> bytes: - return await request.body() - - def _get_node_connection(peer_uid: UID) -> NodeConnection: - # relative - from ..service.network.node_peer import route_to_connection - - network_service = worker.get_service("NetworkService") - peer = network_service.stash.get_by_uid(worker.verify_key, peer_uid).ok() - peer_node_route = peer.pick_highest_priority_route() - connection = route_to_connection(route=peer_node_route) - return connection - - @router.get("/stream/{peer_uid}/{url_path}/", name="stream") - async def stream_download(peer_uid: str, url_path: str) -> StreamingResponse: - try: - url_path_parsed = base64.urlsafe_b64decode(url_path.encode()).decode() - except binascii.Error: - raise HTTPException(404, "Invalid `url_path`.") - - peer_uid_parsed = UID.from_string(peer_uid) - - try: - peer_connection = _get_node_connection(peer_uid_parsed) - url = peer_connection.to_blob_route(url_path_parsed) - stream_response = peer_connection._make_get(url.path, stream=True) - except requests.RequestException: - raise HTTPException(404, "Failed to retrieve data from domain.") - - return StreamingResponse(stream_response, media_type="text/event-stream") - - async def read_request_body_in_chunks( - request: Request, - ) -> AsyncGenerator[bytes, None]: - async for chunk in request.stream(): - yield chunk - - @router.put("/stream/{peer_uid}/{url_path}/", name="stream") - async def stream_upload(peer_uid: str, url_path: str, request: Request) -> Response: - try: - url_path_parsed = base64.urlsafe_b64decode(url_path.encode()).decode() - except binascii.Error: - raise HTTPException(404, "Invalid `url_path`.") - - data = await request.body() - - peer_uid_parsed = UID.from_string(peer_uid) - - try: - peer_connection = _get_node_connection(peer_uid_parsed) - url = peer_connection.to_blob_route(url_path_parsed) - - print("Url on stream", url.path) - response = peer_connection._make_put(url.path, data=data, stream=True) - except requests.RequestException: - raise HTTPException(404, "Failed to upload data to domain") - - return Response( - content=response.content, - headers=response.headers, - media_type="application/octet-stream", - ) - - @router.get( - "/", - name="healthcheck", - status_code=200, - response_class=JSONResponse, - ) - def root() -> dict[str, str]: - """ - Currently, all service backends must satisfy either of the following requirements to - pass the HTTP health checks sent to it from the GCE loadbalancer: 1. Respond with a - 200 on '/'. The content does not matter. 2. Expose an arbitrary url as a readiness - probe on the pods backing the Service. - """ - return {"status": "ok"} - - # provide information about the node in JSON - @router.get("/metadata", response_class=JSONResponse) - def syft_metadata() -> JSONResponse: - return worker.metadata.to(NodeMetadataJSON) - - @router.get("/metadata_capnp") - def syft_metadata_capnp() -> Response: - result = worker.metadata - return Response( - serialize(result, to_bytes=True), - media_type="application/octet-stream", - ) - - def handle_syft_new_api( - user_verify_key: SyftVerifyKey, communication_protocol: PROTOCOL_TYPE - ) -> Response: - return Response( - serialize( - worker.get_api(user_verify_key, communication_protocol), to_bytes=True - ), - media_type="application/octet-stream", - ) - - # get the SyftAPI object - @router.get("/api") - def syft_new_api( - request: Request, verify_key: str, communication_protocol: PROTOCOL_TYPE - ) -> Response: - user_verify_key: SyftVerifyKey = SyftVerifyKey.from_string(verify_key) - if TRACE_MODE: - with trace.get_tracer(syft_new_api.__module__).start_as_current_span( - syft_new_api.__qualname__, - context=extract(request.headers), - kind=trace.SpanKind.SERVER, - ): - return handle_syft_new_api(user_verify_key, communication_protocol) - else: - return handle_syft_new_api(user_verify_key, communication_protocol) - - def handle_new_api_call(data: bytes) -> Response: - obj_msg = deserialize(blob=data, from_bytes=True) - result = worker.handle_api_call(api_call=obj_msg) - return Response( - serialize(result, to_bytes=True), - media_type="application/octet-stream", - ) - - # make a request to the SyftAPI - @router.post("/api_call") - def syft_new_api_call( - request: Request, data: Annotated[bytes, Depends(get_body)] - ) -> Response: - if TRACE_MODE: - with trace.get_tracer(syft_new_api_call.__module__).start_as_current_span( - syft_new_api_call.__qualname__, - context=extract(request.headers), - kind=trace.SpanKind.SERVER, - ): - return handle_new_api_call(data) - else: - return handle_new_api_call(data) - - def handle_login(email: str, password: str, node: AbstractNode) -> Response: - try: - login_credentials = UserLoginCredentials(email=email, password=password) - except ValidationError as e: - return {"Error": e.json()} - - method = node.get_service_method(UserService.exchange_credentials) - context = UnauthedServiceContext(node=node, login_credentials=login_credentials) - result = method(context=context) - - if isinstance(result, SyftError): - logger.error(f"Login Error: {result.message}. user={email}") - response = result - else: - user_private_key = result - if not isinstance(user_private_key, UserPrivateKey): - raise Exception(f"Incorrect return type: {type(user_private_key)}") - response = user_private_key - - return Response( - serialize(response, to_bytes=True), - media_type="application/octet-stream", - ) - - def handle_register(data: bytes, node: AbstractNode) -> Response: - user_create = deserialize(data, from_bytes=True) - - if not isinstance(user_create, UserCreate): - raise Exception(f"Incorrect type received: {user_create}") - - context = NodeServiceContext(node=node) - method = node.get_method_with_context(UserService.register, context) - - result = method(new_user=user_create) - - if isinstance(result, SyftError): - logger.error( - f"Register Error: {result.message}. user={user_create.model_dump()}" - ) - response = SyftError(message=f"{result.message}") - else: - response = result - - return Response( - serialize(response, to_bytes=True), - media_type="application/octet-stream", - ) - - # exchange email and password for a SyftSigningKey - @router.post("/login", name="login", status_code=200) - def login( - request: Request, - email: Annotated[str, Body(example="info@openmined.org")], - password: Annotated[str, Body(example="changethis")], - ) -> Response: - if TRACE_MODE: - with trace.get_tracer(login.__module__).start_as_current_span( - login.__qualname__, - context=extract(request.headers), - kind=trace.SpanKind.SERVER, - ): - return handle_login(email, password, worker) - else: - return handle_login(email, password, worker) - - @router.post("/register", name="register", status_code=200) - def register( - request: Request, data: Annotated[bytes, Depends(get_body)] - ) -> Response: - if TRACE_MODE: - with trace.get_tracer(register.__module__).start_as_current_span( - register.__qualname__, - context=extract(request.headers), - kind=trace.SpanKind.SERVER, - ): - return handle_register(data, worker) - else: - return handle_register(data, worker) - - return router diff --git a/packages/syft/src/syft/node/run.py b/packages/syft/src/syft/node/run.py deleted file mode 100644 index 9fc4878a983..00000000000 --- a/packages/syft/src/syft/node/run.py +++ /dev/null @@ -1,100 +0,0 @@ -# stdlib -import argparse - -# relative -from ..orchestra import Orchestra -from ..orchestra import ServerHandle - - -def str_to_bool(bool_str: str | None) -> bool: - result = False - bool_str = str(bool_str).lower() - if bool_str == "true" or bool_str == "1": - result = True - return result - - -def run() -> ServerHandle | None: - parser = argparse.ArgumentParser() - parser.add_argument("command", help="command: launch", type=str, default="none") - parser.add_argument( - "--name", help="server name", type=str, default="syft-server", dest="name" - ) - parser.add_argument( - "--server-type", - help="server type", - type=str, - default="python", - dest="server_type", - ) - parser.add_argument( - "--host", - help="host for binding", - type=str, - default="0.0.0.0", # nosec - dest="host", - ) - - parser.add_argument( - "--port", help="port for binding", type=int, default=8080, dest="port" - ) - parser.add_argument( - "--dev-mode", - help="developer mode", - type=str, - default="True", - dest="dev_mode", - ) - parser.add_argument( - "--reset", - help="reset", - type=str, - default="True", - dest="reset", - ) - parser.add_argument( - "--local-db", - help="reset", - type=str, - default="False", - dest="local_db", - ) - parser.add_argument( - "--processes", - help="processing mode", - type=int, - default=0, - dest="processes", - ) - parser.add_argument( - "--tail", - help="tail mode", - type=str, - default="True", - dest="tail", - ) - - args = parser.parse_args() - if args.command != "launch": - print("syft launch is the only command currently supported") - - args.dev_mode = str_to_bool(args.dev_mode) - args.reset = str_to_bool(args.reset) - args.local_db = str_to_bool(args.local_db) - args.tail = str_to_bool(args.tail) - args.cmd = str_to_bool(args.cmd) - - server = Orchestra.launch( - name=args.name, - server_type=args.server_type, - host=args.host, - port=args.port, - dev_mode=args.dev_mode, - reset=args.reset, - local_db=args.local_db, - processes=args.processes, - tail=args.tail, - ) - if not args.tail: - return server - return None diff --git a/packages/syft/src/syft/node/server.py b/packages/syft/src/syft/node/server.py deleted file mode 100644 index a3451f304a7..00000000000 --- a/packages/syft/src/syft/node/server.py +++ /dev/null @@ -1,328 +0,0 @@ -# stdlib -from collections.abc import Callable -import multiprocessing -import multiprocessing.synchronize -import os -from pathlib import Path -import platform -import signal -import subprocess # nosec -import sys -import time -from typing import Any - -# third party -from fastapi import APIRouter -from fastapi import FastAPI -from pydantic_settings import BaseSettings -from pydantic_settings import SettingsConfigDict -import requests -from starlette.middleware.cors import CORSMiddleware -import uvicorn - -# relative -from ..abstract_node import NodeSideType -from ..client.client import API_PATH -from ..util.autoreload import enable_autoreload -from ..util.constants import DEFAULT_TIMEOUT -from ..util.util import os_name -from .domain import Domain -from .enclave import Enclave -from .gateway import Gateway -from .node import NodeType -from .routes import make_routes -from .utils import get_named_node_uid -from .utils import remove_temp_dir_for_node - -if os_name() == "macOS": - # needed on MacOS to prevent [__NSCFConstantString initialize] may have been in - # progress in another thread when fork() was called. - multiprocessing.set_start_method("spawn", True) - -WAIT_TIME_SECONDS = 20 - - -class AppSettings(BaseSettings): - name: str - node_type: NodeType = NodeType.DOMAIN - node_side_type: NodeSideType = NodeSideType.HIGH_SIDE - processes: int = 1 - reset: bool = False - dev_mode: bool = False - enable_warnings: bool = False - in_memory_workers: bool = True - queue_port: int | None = None - create_producer: bool = False - n_consumers: int = 0 - association_request_auto_approval: bool = False - background_tasks: bool = False - - model_config = SettingsConfigDict(env_prefix="SYFT_", env_parse_none_str="None") - - -def app_factory() -> FastAPI: - settings = AppSettings() - - worker_classes = { - NodeType.DOMAIN: Domain, - NodeType.GATEWAY: Gateway, - NodeType.ENCLAVE: Enclave, - } - if settings.node_type not in worker_classes: - raise NotImplementedError(f"node_type: {settings.node_type} is not supported") - worker_class = worker_classes[settings.node_type] - - kwargs = settings.model_dump() - if settings.dev_mode: - print( - f"WARN: private key is based on node name: {settings.name} in dev_mode. " - "Don't run this in production." - ) - worker = worker_class.named(**kwargs) - else: - worker = worker_class(**kwargs) - - app = FastAPI(title=settings.name) - router = make_routes(worker=worker) - api_router = APIRouter() - api_router.include_router(router) - app.include_router(api_router, prefix="/api/v2") - app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) - return app - - -def attach_debugger() -> None: - # third party - import debugpy - - os.environ["PYDEVD_DISABLE_FILE_VALIDATION"] = "1" - _, debug_port = debugpy.listen(0) - print( - "\nStarting the server with the Python Debugger enabled (`debug=True`).\n" - 'To attach the debugger, open the command palette in VSCode and select "Debug: Start Debugging (F5)".\n' - f"Then, enter `{debug_port}` in the port field and press Enter.\n", - flush=True, - ) - print(f"Waiting for debugger to attach on port `{debug_port}`...", flush=True) - debugpy.wait_for_client() # blocks execution until a remote debugger is attached - print("Debugger attached", flush=True) - - -def run_uvicorn( - host: str, - port: int, - starting_uvicorn_event: multiprocessing.synchronize.Event, - **kwargs: Any, -) -> None: - should_reset = kwargs.get("dev_mode") and kwargs.get("reset") - - if should_reset: - print("Found `reset=True` in the launch configuration. Resetting the node...") - named_node_uid = get_named_node_uid(kwargs.get("name")) - remove_temp_dir_for_node(named_node_uid) - # Explicitly set `reset` to False to prevent multiple resets during hot-reload - kwargs["reset"] = False - # Kill all old python processes - try: - python_pids = find_python_processes_on_port(port) - for pid in python_pids: - print(f"Stopping process on port: {port}") - kill_process(pid) - time.sleep(1) - except Exception: # nosec - print(f"Failed to kill python process on port: {port}") - - if kwargs.get("debug"): - attach_debugger() - - # Set up all kwargs as environment variables so that they can be accessed in the app_factory function. - env_prefix = AppSettings.model_config.get("env_prefix", "") - for key, value in kwargs.items(): - key_with_prefix = f"{env_prefix}{key.upper()}" - os.environ[key_with_prefix] = str(value) - - # The `serve_node` function calls `run_uvicorn` in a separate process using `multiprocessing.Process`. - # When the child process is created, it inherits the file descriptors from the parent process. - # If the parent process has a file descriptor open for sys.stdin, the child process will also have a file descriptor - # open for sys.stdin. This can cause an OSError in uvicorn when it tries to access sys.stdin in the child process. - # To prevent this, we set sys.stdin to None in the child process. This is safe because we don't actually need - # sys.stdin while running uvicorn programmatically. - sys.stdin = None # type: ignore - - # Signal the parent process that we are starting the uvicorn server. - starting_uvicorn_event.set() - - # Finally, run the uvicorn server. - uvicorn.run( - "syft.node.server:app_factory", - host=host, - port=port, - factory=True, - reload=kwargs.get("dev_mode"), - reload_dirs=[Path(__file__).parent.parent] if kwargs.get("dev_mode") else None, - ) - - -def serve_node( - name: str, - node_type: NodeType = NodeType.DOMAIN, - node_side_type: NodeSideType = NodeSideType.HIGH_SIDE, - host: str = "0.0.0.0", # nosec - port: int = 8080, - processes: int = 1, - reset: bool = False, - dev_mode: bool = False, - tail: bool = False, - enable_warnings: bool = False, - in_memory_workers: bool = True, - queue_port: int | None = None, - create_producer: bool = False, - n_consumers: int = 0, - association_request_auto_approval: bool = False, - background_tasks: bool = False, - debug: bool = False, -) -> tuple[Callable, Callable]: - starting_uvicorn_event = multiprocessing.Event() - - # Enable IPython autoreload if dev_mode is enabled. - if dev_mode: - enable_autoreload() - - server_process = multiprocessing.Process( - target=run_uvicorn, - kwargs={ - "name": name, - "node_type": node_type, - "host": host, - "port": port, - "processes": processes, - "reset": reset, - "dev_mode": dev_mode, - "node_side_type": node_side_type, - "enable_warnings": enable_warnings, - "in_memory_workers": in_memory_workers, - "queue_port": queue_port, - "create_producer": create_producer, - "n_consumers": n_consumers, - "association_request_auto_approval": association_request_auto_approval, - "background_tasks": background_tasks, - "debug": debug, - "starting_uvicorn_event": starting_uvicorn_event, - }, - ) - - def stop() -> None: - print(f"Stopping {name}") - server_process.terminate() - server_process.join(3) - if server_process.is_alive(): - # this is needed because often the process is still alive - server_process.kill() - print("killed") - - def start() -> None: - print(f"Starting {name} server on {host}:{port}") - server_process.start() - - # Wait for the child process to start uvicorn server before starting the readiness checks. - starting_uvicorn_event.wait() - - if tail: - try: - while True: - time.sleep(1) - except KeyboardInterrupt: - try: - stop() - except SystemExit: - os._exit(130) - else: - for i in range(WAIT_TIME_SECONDS): - try: - req = requests.get( - f"http://{host}:{port}{API_PATH}/metadata", - timeout=DEFAULT_TIMEOUT, - ) - if req.status_code == 200: - print(" Done.") - break - except Exception: - time.sleep(1) - if i == 0: - print("Waiting for server to start", end="") - else: - print(".", end="") - - return start, stop - - -def find_python_processes_on_port(port: int) -> list[int]: - system = platform.system() - - if system == "Windows": - command = f"netstat -ano | findstr :{port}" - process = subprocess.Popen( # nosec - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - output, _ = process.communicate() - pids = [ - int(line.strip().split()[-1]) for line in output.split("\n") if line.strip() - ] - - else: # Linux and MacOS - command = f"lsof -i :{port} -sTCP:LISTEN -t" - process = subprocess.Popen( # nosec - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - output, _ = process.communicate() - pids = [int(pid.strip()) for pid in output.split("\n") if pid.strip()] - - python_pids = [] - for pid in pids: - if system == "Windows": - command = ( - f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine" - ) - else: - command = f"ps -p {pid} -o pid,command" - - try: - process = subprocess.Popen( # nosec - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - output, _ = process.communicate() - except Exception as e: - print(f"Error checking process {pid}: {e}") - continue - - lines = output.strip().split("\n") - if len(lines) > 1 and "python" in lines[1].lower(): - python_pids.append(pid) - - return python_pids - - -def kill_process(pid: int) -> None: - try: - os.kill(pid, signal.SIGTERM) - print(f"Process {pid} terminated.") - except Exception as e: - print(f"Error killing process {pid}: {e}") diff --git a/packages/syft/src/syft/node/service_registry.py b/packages/syft/src/syft/node/service_registry.py deleted file mode 100644 index 0d3c514394c..00000000000 --- a/packages/syft/src/syft/node/service_registry.py +++ /dev/null @@ -1,135 +0,0 @@ -# stdlib -from collections.abc import Callable -from dataclasses import dataclass -from dataclasses import field -import typing -from typing import Any -from typing import TYPE_CHECKING - -# relative -from ..serde.serializable import serializable -from ..service.action.action_service import ActionService -from ..service.action.action_store import ActionStore -from ..service.api.api_service import APIService -from ..service.attestation.attestation_service import AttestationService -from ..service.blob_storage.service import BlobStorageService -from ..service.code.status_service import UserCodeStatusService -from ..service.code.user_code_service import UserCodeService -from ..service.code_history.code_history_service import CodeHistoryService -from ..service.data_subject.data_subject_member_service import DataSubjectMemberService -from ..service.data_subject.data_subject_service import DataSubjectService -from ..service.dataset.dataset_service import DatasetService -from ..service.enclave.enclave_service import EnclaveService -from ..service.job.job_service import JobService -from ..service.log.log_service import LogService -from ..service.metadata.metadata_service import MetadataService -from ..service.migration.migration_service import MigrationService -from ..service.network.network_service import NetworkService -from ..service.notification.notification_service import NotificationService -from ..service.notifier.notifier_service import NotifierService -from ..service.output.output_service import OutputService -from ..service.policy.policy_service import PolicyService -from ..service.project.project_service import ProjectService -from ..service.queue.queue_service import QueueService -from ..service.request.request_service import RequestService -from ..service.service import AbstractService -from ..service.settings.settings_service import SettingsService -from ..service.sync.sync_service import SyncService -from ..service.user.user_service import UserService -from ..service.worker.image_registry_service import SyftImageRegistryService -from ..service.worker.worker_image_service import SyftWorkerImageService -from ..service.worker.worker_pool_service import SyftWorkerPoolService -from ..service.worker.worker_service import WorkerService - -if TYPE_CHECKING: - # relative - from .node import Node - - -@serializable() -@dataclass -class ServiceRegistry: - action: ActionService - user: UserService - attestation: AttestationService - worker: WorkerService - settings: SettingsService - dataset: DatasetService - user_code: UserCodeService - log: LogService - request: RequestService - queue: QueueService - job: JobService - api: APIService - data_subject: DataSubjectService - network: NetworkService - policy: PolicyService - notifier: NotifierService - notification: NotificationService - data_subject_member: DataSubjectMemberService - project: ProjectService - enclave: EnclaveService - code_history: CodeHistoryService - metadata: MetadataService - blob_storage: BlobStorageService - migration: MigrationService - syft_worker_image: SyftWorkerImageService - syft_worker_pool: SyftWorkerPoolService - syft_image_registry: SyftImageRegistryService - sync: SyncService - output: OutputService - user_code_status: UserCodeStatusService - - services: list[AbstractService] = field(default_factory=list, init=False) - service_path_map: dict[str, AbstractService] = field( - default_factory=dict, init=False - ) - - @classmethod - def for_node(cls, node: "Node") -> "ServiceRegistry": - return cls(**cls._construct_services(node)) - - def __post_init__(self) -> None: - for name, service_cls in self.get_service_classes().items(): - service = getattr(self, name) - self.services.append(service) - self.service_path_map[service_cls.__name__.lower()] = service - - @classmethod - def get_service_classes( - cls, - ) -> dict[str, type[AbstractService]]: - return { - name: cls - for name, cls in typing.get_type_hints(cls).items() - if issubclass(cls, AbstractService) - } - - @classmethod - def _construct_services(cls, node: "Node") -> dict[str, AbstractService]: - service_dict = {} - for field_name, service_cls in cls.get_service_classes().items(): - svc_kwargs: dict[str, Any] = {} - if issubclass(service_cls.store_type, ActionStore): - svc_kwargs["store"] = node.action_store - else: - svc_kwargs["store"] = node.document_store - - service = service_cls(**svc_kwargs) - service_dict[field_name] = service - return service_dict - - def get_service(self, path_or_func: str | Callable) -> AbstractService: - if callable(path_or_func): - path_or_func = path_or_func.__qualname__ - return self._get_service_from_path(path_or_func) - - def _get_service_from_path(self, path: str) -> AbstractService: - try: - path_list = path.split(".") - if len(path_list) > 1: - _ = path_list.pop() - service_name = path_list.pop() - return self.service_path_map[service_name.lower()] - except KeyError: - raise ValueError(f"Service {path} not found.") diff --git a/packages/syft/src/syft/node/utils.py b/packages/syft/src/syft/node/utils.py deleted file mode 100644 index 3048fd3fa94..00000000000 --- a/packages/syft/src/syft/node/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -# future -from __future__ import annotations - -# stdlib -import os -from pathlib import Path -import shutil -import tempfile - -# relative -from ..types.uid import UID - - -def get_named_node_uid(name: str) -> UID: - """ - Get a unique identifier for a named node. - """ - return UID.with_seed(name) - - -def get_temp_dir_for_node(node_uid: UID, dir_name: str = "") -> Path: - """ - Get a temporary directory unique to the node. - Provide all dbs, blob dirs, and locks using this directory. - """ - root = os.getenv("SYFT_TEMP_ROOT", "syft") - p = Path(tempfile.gettempdir(), root, str(node_uid), dir_name) - p.mkdir(parents=True, exist_ok=True) - return p - - -def remove_temp_dir_for_node(node_uid: UID) -> None: - """ - Remove the temporary directory for this node. - """ - rootdir = get_temp_dir_for_node(node_uid) - if rootdir.exists(): - shutil.rmtree(rootdir, ignore_errors=True) diff --git a/packages/syft/src/syft/node/worker.py b/packages/syft/src/syft/node/worker.py deleted file mode 100644 index 60d3bd61682..00000000000 --- a/packages/syft/src/syft/node/worker.py +++ /dev/null @@ -1,8 +0,0 @@ -# relative -from ..serde.serializable import serializable -from .node import Node - - -@serializable() -class Worker(Node): - pass diff --git a/packages/syft/src/syft/node/worker_settings.py b/packages/syft/src/syft/node/worker_settings.py deleted file mode 100644 index c3b8954a3e8..00000000000 --- a/packages/syft/src/syft/node/worker_settings.py +++ /dev/null @@ -1,52 +0,0 @@ -# future -from __future__ import annotations - -# third party -from typing_extensions import Self - -# relative -from ..abstract_node import AbstractNode -from ..abstract_node import NodeSideType -from ..abstract_node import NodeType -from ..node.credentials import SyftSigningKey -from ..serde.serializable import serializable -from ..service.queue.base_queue import QueueConfig -from ..store.blob_storage import BlobStorageConfig -from ..store.document_store import StoreConfig -from ..types.syft_object import SYFT_OBJECT_VERSION_3 -from ..types.syft_object import SyftObject -from ..types.uid import UID - - -@serializable() -class WorkerSettings(SyftObject): - __canonical_name__ = "WorkerSettings" - __version__ = SYFT_OBJECT_VERSION_3 - - id: UID - name: str - node_type: NodeType - node_side_type: NodeSideType - signing_key: SyftSigningKey - document_store_config: StoreConfig - action_store_config: StoreConfig - blob_store_config: BlobStorageConfig | None = None - queue_config: QueueConfig | None = None - - @classmethod - def from_node(cls, node: AbstractNode) -> Self: - if node.node_side_type: - node_side_type: str = node.node_side_type.value - else: - node_side_type = NodeSideType.HIGH_SIDE - return cls( - id=node.id, - name=node.name, - node_type=node.node_type, - signing_key=node.signing_key, - document_store_config=node.document_store_config, - action_store_config=node.action_store_config, - node_side_type=node_side_type, - blob_store_config=node.blob_store_config, - queue_config=node.queue_config, - ) From b185efb56c1c90e740d551b2813638d0b5220896 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 17:51:43 +0200 Subject: [PATCH 097/123] Update tabulator_template.py --- .../components/tabulator_template.py | 64 +++++++++++++------ 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index cf2d97231bd..f00fed8694f 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -162,15 +162,26 @@ def build_tabulator_table_with_data( max_height: int | None = None, pagination: bool = True, header_sort: bool = True, -) -> str | None: - try: - uid = uid if uid is not None else secrets.token_hex(4) - return _render_tabulator_table( - uid, table_data, table_metadata, max_height, pagination, header_sort - ) - except Exception as e: - logger.debug("Error building table: %s", e) - return None +) -> str: + """ + Builds a Tabulator table for the provided data and metadata. + + Args: + table_data (list[dict]): The data to populate the table. + table_metadata (dict): The metadata for the table. + uid (str, optional): The unique identifier for the table. Defaults to None. + max_height (int, optional): The maximum height of the table. Defaults to None. + pagination (bool, optional): Whether to enable pagination. Defaults to True. + header_sort (bool, optional): Whether to enable header sorting. Defaults to True. + + Returns: + str: The HTML representation of the Tabulator table. + + """ + uid = uid if uid is not None else secrets.token_hex(4) + return _render_tabulator_table( + uid, table_data, table_metadata, max_height, pagination, header_sort + ) def build_tabulator_table( @@ -180,17 +191,32 @@ def build_tabulator_table( pagination: bool = True, header_sort: bool = True, ) -> str | None: - try: - table_data, table_metadata = prepare_table_data(obj) - if len(table_data) == 0: + """ + Builds a Tabulator table from the given object if possible. + + If the object cannot be represented as a table, returns None. + + Args: + obj (Any): The object to build the table from. + uid (str, optional): The unique identifier for the table. Defaults to None. + max_height (int, optional): The maximum height of the table. Defaults to None. + pagination (bool, optional): Whether to enable pagination. Defaults to True. + header_sort (bool, optional): Whether to enable header sorting. Defaults to True. + + Returns: + str | None: The HTML representation of the Tabulator table or None + + """ + table_data, table_metadata = prepare_table_data(obj) + if len(table_data) == 0: + if hasattr(obj, "__len__") and len(obj) == 0: return obj.__repr__() - uid = uid if uid is not None else secrets.token_hex(4) - return _render_tabulator_table( - uid, table_data, table_metadata, max_height, pagination, header_sort - ) - except Exception as e: - logger.debug("error building table", exc_info=e) - return None + else: + return None + + return build_tabulator_table_with_data( + table_data, table_metadata, uid, max_height, pagination, header_sort + ) def show_table(obj: Any) -> None: From f285dc98a19133c25f84c55be77c8098b95c6a99 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 17:52:51 +0200 Subject: [PATCH 098/123] Update user_code_service.py --- .../syft/service/code/user_code_service.py | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 77ade58b889..4d200bdbf22 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -50,7 +50,7 @@ @instrument -@serializable() +@serializable(canonical_name="UserCodeService", version=1) class UserCodeService(AbstractService): store: DocumentStore stash: UserCodeStash @@ -109,11 +109,11 @@ def _submit( # if the validation fails, we should remove the user code status # and code version to prevent dangling status root_context = AuthedServiceContext( - credentials=context.node.verify_key, node=context.node + credentials=context.server.verify_key, server=context.server ) if code.status_link is not None: - _ = context.node.get_service("usercodestatusservice").remove( + _ = context.server.get_service("usercodestatusservice").remove( root_context, code.status_link.object_uid ) return result @@ -189,7 +189,7 @@ def _post_user_code_transform_ops( ): return Err("outputs can only be distributed to input owners") - worker_pool_service = context.node.get_service("SyftWorkerPoolService") + worker_pool_service = context.server.get_service("SyftWorkerPoolService") pool_result = worker_pool_service._get_worker_pool( context, pool_name=user_code.worker_pool_name, @@ -199,7 +199,7 @@ def _post_user_code_transform_ops( return Err(pool_result.message) # Create a code history - code_history_service = context.node.get_service("codehistoryservice") + code_history_service = context.server.get_service("codehistoryservice") result = code_history_service.submit_version(context=context, code=user_code) if isinstance(result, SyftError): return Err(result.message) @@ -213,7 +213,7 @@ def _request_code_execution( reason: str | None = "", ) -> Request | SyftError: # Cannot make multiple requests for the same code - get_by_usercode_id = context.node.get_service_method( + get_by_usercode_id = context.server.get_service_method( RequestService.get_by_usercode_id ) existing_requests = get_by_usercode_id(context, user_code.id) @@ -234,7 +234,7 @@ def _request_code_execution( ] ) - code_link = LinkedObject.from_obj(user_code, node_uid=context.node.id) + code_link = LinkedObject.from_obj(user_code, server_uid=context.server.id) # Requests made on low side are synced, and have their status computed instead of set manually. if user_code.is_l0_deployment: @@ -252,7 +252,7 @@ def _request_code_execution( changes = [status_change] request = SubmitRequest(changes=changes) - method = context.node.get_service_method(RequestService.submit) + method = context.server.get_service_method(RequestService.submit) result = method(context=context, request=request, reason=reason) # The Request service already returns either a SyftSuccess or SyftError @@ -275,7 +275,7 @@ def _get_or_submit_user_code( return user_code_or_err user_code = user_code_or_err.ok() if user_code is None: - return Err("UserCode not found on this node.") + return Err("UserCode not found on this server.") return Ok(user_code) else: # code: SubmitUserCode # Submit new UserCode, or get existing UserCode with the same code hash @@ -324,9 +324,13 @@ def get_by_uid( result = self.stash.get_by_uid(context.credentials, uid=uid) if result.is_ok(): user_code = result.ok() - if user_code and user_code.input_policy_state and context.node is not None: + if ( + user_code + and user_code.input_policy_state + and context.server is not None + ): # TODO replace with LinkedObject Context - user_code.node_uid = context.node.id + user_code.server_uid = context.server.id return user_code return SyftError(message=result.err()) @@ -375,7 +379,7 @@ def is_execution_allowed( elif not (has_code_permission := self.has_code_permission(code, context)): return has_code_permission elif not code.is_output_policy_approved(context): - return SyftError("Output policy not approved", code) + return SyftError(message=f"Output policy not approved. {code}") policy_is_valid = output_policy is not None and output_policy._is_valid(context) if not policy_is_valid: @@ -389,7 +393,7 @@ def is_execution_on_owned_args_allowed( if context.role == ServiceRole.ADMIN: return True - user_service = context.node.get_service("userservice") + user_service = context.server.get_service("userservice") current_user = user_service.get_current_user(context=context) return current_user.mock_execution_permission @@ -398,15 +402,15 @@ def keep_owned_kwargs( ) -> dict[str, Any] | SyftError: """Return only the kwargs that are owned by the user""" - action_service = context.node.get_service("actionservice") + action_service = context.server.get_service("actionservice") mock_kwargs = {} for k, v in kwargs.items(): if isinstance(v, UID): # Jobs have UID kwargs instead of ActionObject - val = action_service.get(context, uid=v) - if val.is_ok(): - v = val.ok() + v = action_service.get(context, uid=v) + if v.is_ok(): + v = v.ok() if ( isinstance(v, ActionObject) and v.syft_client_verify_key == context.credentials @@ -433,8 +437,8 @@ def is_execution_on_owned_args( return False code = code.ok() - # Skip the domain and context kwargs, they are passed by the backend - code_kwargs = set(code.signature.parameters.keys()) - {"domain", "context"} + # Skip the datasite and context kwargs, they are passed by the backend + code_kwargs = set(code.signature.parameters.keys()) - {"datasite", "context"} passed_kwarg_keys = set(passed_kwargs.keys()) return passed_kwarg_keys == code_kwargs @@ -456,7 +460,7 @@ def valid_worker_pool_for_context( ) -> bool: """This is a temporary fix that is needed until every function is always just ran as job""" # relative - from ...node.node import get_default_worker_pool_name + from ...server.server import get_default_worker_pool_name has_custom_worker_pool = ( user_code.worker_pool_name is not None @@ -581,7 +585,7 @@ def _call( "which is currently not supported. Run your function with `blocking=False` to run" " as a job on your worker pool" ) - action_service: ActionService = context.node.get_service("actionservice") # type: ignore + action_service: ActionService = context.server.get_service("actionservice") # type: ignore result_action_object: Result[ActionObject | TwinObject, str] = ( action_service._user_code_execute( context, code, kwarg2id, result_id=result_id @@ -648,9 +652,9 @@ def _call( def has_code_permission( self, code_item: UserCode, context: AuthedServiceContext ) -> SyftSuccess | SyftError: - if context.credentials not in ( - context.node.verify_key, - code_item.user_verify_key, + if not ( + context.credentials == context.server.verify_key + or context.credentials == code_item.user_verify_key ): return SyftError( message=f"Code Execution Permission: {context.credentials} denied" @@ -700,8 +704,8 @@ def resolve_outputs( return None outputs = [] for output_id in output_ids: - if context.node is not None: - action_service = context.node.get_service("actionservice") + if context.server is not None: + action_service = context.server.get_service("actionservice") result = action_service.get( context, uid=output_id, twin_mode=TwinMode.PRIVATE ) From 2e15bbface8664da13e63b1d9553846173f63de9 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 17:54:58 +0200 Subject: [PATCH 099/123] Update and rename node_peer.py to server_peer.py --- .../network/{node_peer.py => server_peer.py} | 209 ++++++++---------- 1 file changed, 89 insertions(+), 120 deletions(-) rename packages/syft/src/syft/service/network/{node_peer.py => server_peer.py} (55%) diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/server_peer.py similarity index 55% rename from packages/syft/src/syft/service/network/node_peer.py rename to packages/syft/src/syft/service/network/server_peer.py index efb3b35c831..941396820d5 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/server_peer.py @@ -9,72 +9,50 @@ from result import Result # relative -from ...abstract_node import NodeType -from ...client.client import NodeConnection +from ...abstract_server import ServerType +from ...client.client import ServerConnection from ...client.client import SyftClient -from ...node.credentials import SyftSigningKey -from ...node.credentials import SyftVerifyKey from ...serde.serializable import serializable +from ...server.credentials import SyftSigningKey +from ...server.credentials import SyftVerifyKey from ...service.response import SyftError from ...types.datetime import DateTime -from ...types.syft_migration import migrate from ...types.syft_object import PartialSyftObject from ...types.syft_object import SYFT_OBJECT_VERSION_1 -from ...types.syft_object import SYFT_OBJECT_VERSION_2 -from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...types.syft_object import SyftObject from ...types.transforms import TransformContext from ...types.uid import UID -from ..context import NodeServiceContext -from ..metadata.node_metadata import NodeMetadata -from .routes import HTTPNodeRoute -from .routes import NodeRoute -from .routes import NodeRouteType -from .routes import NodeRouteTypeV1 -from .routes import PythonNodeRoute -from .routes import VeilidNodeRoute +from ..context import ServerServiceContext +from ..metadata.server_metadata import ServerMetadata +from .routes import HTTPServerRoute +from .routes import PythonServerRoute +from .routes import ServerRoute +from .routes import ServerRouteType +from .routes import VeilidServerRoute from .routes import connection_to_route from .routes import route_to_connection logger = logging.getLogger(__name__) -@serializable() -class NodePeerConnectionStatus(Enum): +@serializable(canonical_name="ServerPeerConnectionStatus", version=1) +class ServerPeerConnectionStatus(Enum): ACTIVE = "ACTIVE" INACTIVE = "INACTIVE" TIMEOUT = "TIMEOUT" @serializable() -class NodePeerV2(SyftObject): +class ServerPeer(SyftObject): # version - __canonical_name__ = "NodePeer" - __version__ = SYFT_OBJECT_VERSION_2 - - __attr_searchable__ = ["name", "node_type"] - __attr_unique__ = ["verify_key"] - __repr_attrs__ = ["name", "node_type", "admin_email"] - - id: UID | None = None # type: ignore[assignment] - name: str - verify_key: SyftVerifyKey - node_routes: list[NodeRouteTypeV1] = [] - node_type: NodeType - admin_email: str - - -@serializable() -class NodePeer(SyftObject): - # version - __canonical_name__ = "NodePeer" - __version__ = SYFT_OBJECT_VERSION_3 + __canonical_name__ = "ServerPeer" + __version__ = SYFT_OBJECT_VERSION_1 - __attr_searchable__ = ["name", "node_type"] + __attr_searchable__ = ["name", "server_type"] __attr_unique__ = ["verify_key"] __repr_attrs__ = [ "name", - "node_type", + "server_type", "admin_email", "ping_status", "ping_status_message", @@ -84,51 +62,53 @@ class NodePeer(SyftObject): id: UID | None = None # type: ignore[assignment] name: str verify_key: SyftVerifyKey - node_routes: list[NodeRouteType] = [] - node_type: NodeType + server_routes: list[ServerRouteType] = [] + server_type: ServerType admin_email: str - ping_status: NodePeerConnectionStatus | None = None + ping_status: ServerPeerConnectionStatus | None = None ping_status_message: str | None = None pinged_timestamp: DateTime | None = None - def existed_route(self, route: NodeRouteType) -> tuple[bool, int | None]: - """Check if a route exists in self.node_routes + def existed_route(self, route: ServerRouteType) -> tuple[bool, int | None]: + """Check if a route exists in self.server_routes Args: route: the route to be checked. For now it can be either - HTTPNodeRoute or PythonNodeRoute + HTTPServerRoute or PythonServerRoute Returns: - if the route exists, returns (True, index of the existed route in self.node_routes) + if the route exists, returns (True, index of the existed route in self.server_routes) if the route does not exist returns (False, None) """ if route: - if not isinstance(route, HTTPNodeRoute | PythonNodeRoute | VeilidNodeRoute): + if not isinstance( + route, HTTPServerRoute | PythonServerRoute | VeilidServerRoute + ): raise ValueError(f"Unsupported route type: {type(route)}") - for i, r in enumerate(self.node_routes): + for i, r in enumerate(self.server_routes): if route == r: return (True, i) return (False, None) - def update_route_priority(self, route: NodeRoute) -> NodeRoute: + def update_route_priority(self, route: ServerRoute) -> ServerRoute: """ Assign the new_route's priority to be current max + 1 Args: - route (NodeRoute): The new route whose priority is to be updated. + route (ServerRoute): The new route whose priority is to be updated. Returns: - NodeRoute: The new route with the updated priority + ServerRoute: The new route with the updated priority """ - current_max_priority: int = max(route.priority for route in self.node_routes) + current_max_priority: int = max(route.priority for route in self.server_routes) route.priority = current_max_priority + 1 return route - def pick_highest_priority_route(self, oldest: bool = True) -> NodeRoute: + def pick_highest_priority_route(self, oldest: bool = True) -> ServerRoute: """ - Picks the route with the highest priority from the list of node routes. + Picks the route with the highest priority from the list of server routes. Args: oldest (bool): @@ -138,46 +118,47 @@ def pick_highest_priority_route(self, oldest: bool = True) -> NodeRoute: meaning the route with max priority value. Returns: - NodeRoute: The route with the highest priority. + ServerRoute: The route with the highest priority. """ - highest_priority_route: NodeRoute = self.node_routes[-1] - for route in self.node_routes[:-1]: + highest_priority_route: ServerRoute = self.server_routes[-1] + for route in self.server_routes[:-1]: if oldest: if route.priority < highest_priority_route.priority: highest_priority_route = route - elif route.priority > highest_priority_route.priority: - highest_priority_route = route + else: + if route.priority > highest_priority_route.priority: + highest_priority_route = route return highest_priority_route - def update_route(self, route: NodeRoute) -> None: + def update_route(self, route: ServerRoute) -> None: """ - Update the route for the node. + Update the route for the server. If the route already exists, return it. If the route is new, assign it to have the priority of (current_max + 1) Args: - route (NodeRoute): The new route to be added to the peer. + route (ServerRoute): The new route to be added to the peer. """ existed, idx = self.existed_route(route) if existed: - self.node_routes[idx] = route # type: ignore + self.server_routes[idx] = route # type: ignore else: new_route = self.update_route_priority(route) - self.node_routes.append(new_route) + self.server_routes.append(new_route) - def update_routes(self, new_routes: list[NodeRoute]) -> None: + def update_routes(self, new_routes: list[ServerRoute]) -> None: """ - Update multiple routes of the node peer. + Update multiple routes of the server peer. This method takes a list of new routes as input. It first updates the priorities of the new routes. - Then, for each new route, it checks if the route already exists for the node peer. + Then, for each new route, it checks if the route already exists for the server peer. If it does, it updates the priority of the existing route. - If it doesn't, it adds the new route to the node. + If it doesn't, it adds the new route to the server. Args: - new_routes (list[NodeRoute]): The new routes to be added to the node. + new_routes (list[ServerRoute]): The new routes to be added to the server. Returns: None @@ -186,18 +167,18 @@ def update_routes(self, new_routes: list[NodeRoute]) -> None: self.update_route(new_route) def update_existed_route_priority( - self, route: NodeRoute, priority: int | None = None - ) -> NodeRouteType | SyftError: + self, route: ServerRoute, priority: int | None = None + ) -> ServerRouteType | SyftError: """ Update the priority of an existed route. Args: - route (NodeRoute): The route whose priority is to be updated. + route (ServerRoute): The route whose priority is to be updated. priority (int | None): The new priority of the route. If not given, the route will be assigned with the highest priority. Returns: - NodeRoute: The route with updated priority if the route exists + ServerRoute: The route with updated priority if the route exists SyftError: If the route does not exist or the priority is invalid """ if priority is not None and priority <= 0: @@ -211,63 +192,61 @@ def update_existed_route_priority( return SyftError(message=f"Route with id {route.id} does not exist.") if priority is not None: - self.node_routes[index].priority = priority + self.server_routes[index].priority = priority else: - self.node_routes[index].priority = self.update_route_priority( + self.server_routes[index].priority = self.update_route_priority( route ).priority - return self.node_routes[index] + return self.server_routes[index] @staticmethod - def from_client(client: SyftClient) -> "NodePeer": + def from_client(client: SyftClient) -> "ServerPeer": if not client.metadata: raise ValueError("Client has to have metadata first") - peer = client.metadata.to(NodeMetadata).to(NodePeer) + peer = client.metadata.to(ServerMetadata).to(ServerPeer) route = connection_to_route(client.connection) - peer.node_routes.append(route) + peer.server_routes.append(route) return peer @property - def latest_added_route(self) -> NodeRoute | None: + def latest_added_route(self) -> ServerRoute | None: """ - Returns the latest added route from the list of node routes. + Returns the latest added route from the list of server routes. Returns: - NodeRoute | None: The latest added route, or None if there are no routes. + ServerRoute | None: The latest added route, or None if there are no routes. """ - return self.node_routes[-1] if self.node_routes else None + return self.server_routes[-1] if self.server_routes else None def client_with_context( - self, context: NodeServiceContext + self, context: ServerServiceContext ) -> Result[type[SyftClient], str]: # third party - if len(self.node_routes) < 1: + if len(self.server_routes) < 1: raise ValueError(f"No routes to peer: {self}") # select the route with highest priority to connect to the peer - final_route: NodeRoute = self.pick_highest_priority_route() - connection: NodeConnection = route_to_connection(route=final_route) + final_route: ServerRoute = self.pick_highest_priority_route() + connection: ServerConnection = route_to_connection(route=final_route) try: client_type = connection.get_client_type() except Exception as e: - msg = ( - f"Failed to establish a connection with {self.node_type} '{self.name}'" - ) + msg = f"Failed to establish a connection with {self.server_type} '{self.name}'" logger.error(msg, exc_info=e) return Err(msg) if isinstance(client_type, SyftError): return Err(client_type.message) return Ok( - client_type(connection=connection, credentials=context.node.signing_key) + client_type(connection=connection, credentials=context.server.signing_key) ) def client_with_key(self, credentials: SyftSigningKey) -> SyftClient | SyftError: - if len(self.node_routes) < 1: + if len(self.server_routes) < 1: raise ValueError(f"No routes to peer: {self}") - final_route: NodeRoute = self.pick_highest_priority_route() + final_route: ServerRoute = self.pick_highest_priority_route() connection = route_to_connection(route=final_route) client_type = connection.get_client_type() @@ -284,26 +263,26 @@ def guest_client(self) -> SyftClient: def proxy_from(self, client: SyftClient) -> SyftClient: return client.proxy_to(self) - def get_rtunnel_route(self) -> HTTPNodeRoute | None: - for route in self.node_routes: + def get_rtunnel_route(self) -> HTTPServerRoute | None: + for route in self.server_routes: if hasattr(route, "rtunnel_token") and route.rtunnel_token: return route return None - def delete_route(self, route: NodeRouteType) -> SyftError | None: + def delete_route(self, route: ServerRouteType) -> SyftError | None: """ Deletes a route from the peer's route list. - Takes O(n) where is n is the number of routes in self.node_routes. + Takes O(n) where is n is the number of routes in self.server_routes. Args: - route (NodeRouteType): The route to be deleted; + route (ServerRouteType): The route to be deleted; Returns: - SyftError: If failing to delete node route + SyftError: If failing to delete server route """ if route: try: - self.node_routes = [r for r in self.node_routes if r != route] + self.server_routes = [r for r in self.server_routes if r != route] except Exception as e: return SyftError( message=f"Error deleting route with id {route.id}. Exception: {e}" @@ -313,15 +292,15 @@ def delete_route(self, route: NodeRouteType) -> SyftError | None: @serializable() -class NodePeerUpdate(PartialSyftObject): - __canonical_name__ = "NodePeerUpdate" +class ServerPeerUpdate(PartialSyftObject): + __canonical_name__ = "ServerPeerUpdate" __version__ = SYFT_OBJECT_VERSION_1 id: UID name: str - node_routes: list[NodeRouteType] + server_routes: list[ServerRouteType] admin_email: str - ping_status: NodePeerConnectionStatus + ping_status: ServerPeerConnectionStatus ping_status_message: str pinged_timestamp: DateTime @@ -329,23 +308,13 @@ class NodePeerUpdate(PartialSyftObject): def drop_veilid_route() -> Callable: def _drop_veilid_route(context: TransformContext) -> TransformContext: if context.output: - node_routes = context.output["node_routes"] + server_routes = context.output["server_routes"] new_routes = [ - node_route - for node_route in node_routes - if not isinstance(node_route, VeilidNodeRoute) + server_route + for server_route in server_routes + if not isinstance(server_route, VeilidServerRoute) ] - context.output["node_routes"] = new_routes + context.output["server_routes"] = new_routes return context return _drop_veilid_route - - -@migrate(NodePeerV2, NodePeer) -def upgrade_node_peer() -> list[Callable]: - return [drop_veilid_route()] - - -@migrate(NodePeerV2, NodePeer) -def downgrade_node_peer() -> list[Callable]: - return [] From 95d04b52be4b66c4de761c4944b723d0ce9714a1 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 17:56:12 +0200 Subject: [PATCH 100/123] Update project_service.py --- .../syft/service/project/project_service.py | 103 +++++++++--------- 1 file changed, 54 insertions(+), 49 deletions(-) diff --git a/packages/syft/src/syft/service/project/project_service.py b/packages/syft/src/syft/service/project/project_service.py index 708cd5db19c..0da9d043e18 100644 --- a/packages/syft/src/syft/service/project/project_service.py +++ b/packages/syft/src/syft/service/project/project_service.py @@ -28,7 +28,7 @@ @instrument -@serializable() +@serializable(canonical_name="ProjectService", version=1) class ProjectService(AbstractService): store: DocumentStore stash: ProjectStash @@ -43,7 +43,7 @@ def __init__(self, store: DocumentStore) -> None: roles=ONLY_DATA_SCIENTIST_ROLE_LEVEL, ) def can_create_project(self, context: AuthedServiceContext) -> bool | SyftError: - user_service = context.node.get_service("userservice") + user_service = context.server.get_service("userservice") role = user_service.get_role_for_credentials(credentials=context.credentials) if role == ServiceRole.DATA_SCIENTIST: return True @@ -66,7 +66,7 @@ def create_project( try: # Check if the project with given id already exists project_id_check = self.stash.get_by_uid( - credentials=context.node.verify_key, uid=project.id + credentials=context.server.verify_key, uid=project.id ) if project_id_check.is_err(): @@ -79,44 +79,49 @@ def create_project( project_obj: Project = project.to(Project, context=context) - # Updating the leader node route of the project object - # In case the current node, is the leader, they would input their node route - # For the followers, they would check if the leader is their node peer + # Updating the leader server route of the project object + # In case the current server, is the leader, they would input their server route + # For the followers, they would check if the leader is their server peer # using the leader's verify_key # If the follower do not have the leader as its peer in its routes # They would raise as error - leader_node = project_obj.state_sync_leader + leader_server = project_obj.state_sync_leader - # If the current node is a follower - # For followers the leader node route is retrieved from its peer - if leader_node.verify_key != context.node.verify_key: - network_service = context.node.get_service("networkservice") + # If the current server is a follower + # For followers the leader server route is retrieved from its peer + if leader_server.verify_key != context.server.verify_key: + network_service = context.server.get_service("networkservice") peer = network_service.stash.get_by_verify_key( - credentials=context.node.verify_key, - verify_key=leader_node.verify_key, + credentials=context.server.verify_key, + verify_key=leader_server.verify_key, ) if peer.is_err(): - this_node_id = context.node.id.short() if context.node.id else "" + this_server_id = ( + context.server.id.short() if context.server.id else "" + ) return SyftError( message=( - f"Leader Node(id={leader_node.id.short()}) is not a " - f"peer of this Node(id={this_node_id})" + f"Leader Server(id={leader_server.id.short()}) is not a " + f"peer of this Server(id={this_server_id})" ) ) - leader_node_peer = peer.ok() - elif project.leader_node_route is not None: - # for the leader node, as it does not have route information to itself + leader_server_peer = peer.ok() + else: + # for the leader server, as it does not have route information to itself # we rely on the data scientist to provide the route # the route is then validated by the leader - leader_node_peer = project.leader_node_route.validate_with_context( - context=context - ) - else: - return SyftError( - message=f"project {project}'s leader_node_route is None" - ) + if project.leader_server_route is not None: + leader_server_peer = ( + project.leader_server_route.validate_with_context( + context=context + ) + ) + else: + return SyftError( + message=f"project {project}'s leader_server_route is None" + ) - project_obj.leader_node_peer = leader_node_peer + project_obj.leader_server_peer = leader_server_peer # This should always be the last call before flushing to DB project_obj.start_hash = create_project_hash(project_obj)[1] @@ -148,15 +153,15 @@ def add_event( # Event object should be received from the leader of the project - # retrieve the project object by node verify key + # retrieve the project object by server verify key project_obj = self.stash.get_by_uid( - context.node.verify_key, uid=project_event.project_id + context.server.verify_key, uid=project_event.project_id ) if project_obj.is_err(): return SyftError(message=str(project_obj.err())) project: Project = project_obj.ok() - if project.state_sync_leader.verify_key == context.node.verify_key: + if project.state_sync_leader.verify_key == context.server.verify_key: return SyftError( message="Project Events should be passed to leader by broadcast endpoint" ) @@ -170,8 +175,8 @@ def add_event( if isinstance(message_result, SyftError): return message_result - # updating the project object using root verify key of node - result = self.stash.update(context.node.verify_key, project) + # updating the project object using root verify key of server + result = self.stash.update(context.server.verify_key, project) if result.is_err(): return SyftError(message=str(result.err())) @@ -193,7 +198,7 @@ def broadcast_event( # The leader broadcasts the event to all the members of the project project_obj = self.stash.get_by_uid( - context.node.verify_key, uid=project_event.project_id + context.server.verify_key, uid=project_event.project_id ) if project_obj.is_err(): @@ -203,7 +208,7 @@ def broadcast_event( if not project.has_permission(context.credentials): return SyftError(message="User does not have permission to add events") - if project.state_sync_leader.verify_key != context.node.verify_key: + if project.state_sync_leader.verify_key != context.server.verify_key: return SyftError( message="Only the leader of the project can broadcast events" ) @@ -223,18 +228,18 @@ def broadcast_event( return message_result # Broadcast the event to all the members of the project - network_service = context.node.get_service("networkservice") + network_service = context.server.get_service("networkservice") for member in project.members: - if member.verify_key != context.node.verify_key: - # Retrieving the NodePeer Object to communicate with the node + if member.verify_key != context.server.verify_key: + # Retrieving the ServerPeer Object to communicate with the server peer = network_service.stash.get_by_verify_key( - credentials=context.node.verify_key, + credentials=context.server.verify_key, verify_key=member.verify_key, ) if peer.is_err(): return SyftError( - message=f"Leader node does not have peer {member.name}-{member.id.short()}" + message=f"Leader server does not have peer {member.name}-{member.id.short()}" + " Kindly exchange routes with the peer" ) peer = peer.ok() @@ -252,7 +257,7 @@ def broadcast_event( if isinstance(event_result, SyftError): return event_result - result = self.stash.update(context.node.verify_key, project) + result = self.stash.update(context.server.verify_key, project) if result.is_err(): return SyftError(message=str(result.err())) @@ -270,13 +275,13 @@ def sync( # Event object should be received from the leader of the project - # retrieve the project object by node verify key - project_obj = self.stash.get_by_uid(context.node.verify_key, uid=project_id) + # retrieve the project object by server verify key + project_obj = self.stash.get_by_uid(context.server.verify_key, uid=project_id) if project_obj.is_err(): return SyftError(message=str(project_obj.err())) project: Project = project_obj.ok() - if project.state_sync_leader.verify_key != context.node.verify_key: + if project.state_sync_leader.verify_key != context.server.verify_key: return SyftError( message="Project Events should be synced only with the leader" ) @@ -333,7 +338,7 @@ def get_by_uid( self, context: AuthedServiceContext, uid: UID ) -> Project | SyftError: result = self.stash.get_by_uid( - credentials=context.node.verify_key, + credentials=context.server.verify_key, uid=uid, ) if result.is_err(): @@ -348,7 +353,7 @@ def add_signing_key_to_project( # Automatically infuse signing key of user # requesting get_all() or creating the project object - user_service = context.node.get_service("userservice") + user_service = context.server.get_service("userservice") user = user_service.stash.get_by_verify_key( credentials=context.credentials, verify_key=context.credentials ) @@ -374,7 +379,7 @@ def check_for_project_request( Args: project (Project): Project object project_event (ProjectEvent): Project event object - context (AuthedServiceContext): Context of the node + context (AuthedServiceContext): Context of the server Returns: Union[SyftSuccess, SyftError]: SyftSuccess if message is created else SyftError @@ -382,16 +387,16 @@ def check_for_project_request( if ( isinstance(project_event, ProjectRequest) - and project_event.linked_request.node_uid == context.node.id + and project_event.linked_request.server_uid == context.server.id ): link = LinkedObject.with_context(project, context=context) message = CreateNotification( subject=f"A new request has been added to the Project: {project.name}.", from_user_verify_key=context.credentials, - to_user_verify_key=context.node.verify_key, + to_user_verify_key=context.server.verify_key, linked_obj=link, ) - method = context.node.get_service_method(NotificationService.send) + method = context.server.get_service_method(NotificationService.send) result = method(context=context, notification=message) if isinstance(result, SyftError): return result From 57d5133d8dcdc0b451c87d27ecb578b7cd2e59b4 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:04:25 +0200 Subject: [PATCH 101/123] Lint PLW1508 --- packages/grid/backend/grid/core/config.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 0d7d090743a..0b4dca89eea 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -108,17 +108,17 @@ def get_emails_enabled(self) -> Self: True if os.getenv("USE_BLOB_STORAGE", "false").lower() == "true" else False ) S3_ENDPOINT: str = os.getenv("S3_ENDPOINT", "seaweedfs") - S3_PORT: int = int(os.getenv("S3_PORT", 8333)) + S3_PORT: int = int(os.getenv("S3_PORT", "8333")) S3_ROOT_USER: str = os.getenv("S3_ROOT_USER", "admin") S3_ROOT_PWD: str | None = os.getenv("S3_ROOT_PWD", "admin") S3_REGION: str = os.getenv("S3_REGION", "us-east-1") S3_PRESIGNED_TIMEOUT_SECS: int = int( - os.getenv("S3_PRESIGNED_TIMEOUT_SECS", 1800) + os.getenv("S3_PRESIGNED_TIMEOUT_SECS", "1800") ) # 30 minutes in seconds - SEAWEED_MOUNT_PORT: int = int(os.getenv("SEAWEED_MOUNT_PORT", 4001)) + SEAWEED_MOUNT_PORT: int = int(os.getenv("SEAWEED_MOUNT_PORT", "4001")) # REDIS_HOST: str = str(os.getenv("REDIS_HOST", "redis")) - # REDIS_PORT: int = int(os.getenv("REDIS_PORT", 6379)) + # REDIS_PORT: int = int(os.getenv("REDIS_PORT", "6379")) # REDIS_STORE_DB_ID: int = int(os.getenv("REDIS_STORE_DB_ID", 0)) # REDIS_LEDGER_DB_ID: int = int(os.getenv("REDIS_LEDGER_DB_ID", 1)) # STORE_DB_ID: int = int(os.getenv("STORE_DB_ID", 0)) @@ -127,25 +127,25 @@ def get_emails_enabled(self) -> Self: # DATASITE_CHECK_INTERVAL: int = int(os.getenv("DATASITE_CHECK_INTERVAL", 60)) CONTAINER_HOST: str = str(os.getenv("CONTAINER_HOST", "docker")) MONGO_HOST: str = str(os.getenv("MONGO_HOST", "")) - MONGO_PORT: int = int(os.getenv("MONGO_PORT", 27017)) + MONGO_PORT: int = int(os.getenv("MONGO_PORT", "27017")) MONGO_USERNAME: str = str(os.getenv("MONGO_USERNAME", "")) MONGO_PASSWORD: str = str(os.getenv("MONGO_PASSWORD", "")) DEV_MODE: bool = True if os.getenv("DEV_MODE", "false").lower() == "true" else False # ZMQ stuff - QUEUE_PORT: int = int(os.getenv("QUEUE_PORT", 5556)) + QUEUE_PORT: int = int(os.getenv("QUEUE_PORT", "5556")) CREATE_PRODUCER: bool = ( True if os.getenv("CREATE_PRODUCER", "false").lower() == "true" else False ) - N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", 1)) + N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", "1")) SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", False)) CONSUMER_SERVICE_NAME: str | None = os.getenv("CONSUMER_SERVICE_NAME") - INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", True)) + INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", "True")) SMTP_USERNAME: str = os.getenv("SMTP_USERNAME", "") EMAIL_SENDER: str = os.getenv("EMAIL_SENDER", "") SMTP_PASSWORD: str = os.getenv("SMTP_PASSWORD", "") SMTP_TLS: bool = True - SMTP_PORT: int = int(os.getenv("SMTP_PORT", 587)) + SMTP_PORT: int = int(os.getenv("SMTP_PORT", "587")) SMTP_HOST: str = os.getenv("SMTP_HOST", "") TEST_MODE: bool = ( @@ -155,7 +155,7 @@ def get_emails_enabled(self) -> Self: ASSOCIATION_REQUEST_AUTO_APPROVAL: bool = str_to_bool( os.getenv("ASSOCIATION_REQUEST_AUTO_APPROVAL", "False") ) - MIN_SIZE_BLOB_STORAGE_MB: int = int(os.getenv("MIN_SIZE_BLOB_STORAGE_MB", 16)) + MIN_SIZE_BLOB_STORAGE_MB: int = int(os.getenv("MIN_SIZE_BLOB_STORAGE_MB", "16")) REVERSE_TUNNEL_ENABLED: bool = str_to_bool( os.getenv("REVERSE_TUNNEL_ENABLED", "false") ) From 277b0dcd5decb3d654964a8d3ac5d7561bb8960e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:06:30 +0200 Subject: [PATCH 102/123] Lint PLW1508 --- packages/grid/backend/grid/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 0b4dca89eea..26ec2c94a1a 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -138,7 +138,7 @@ def get_emails_enabled(self) -> Self: ) N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", "1")) SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") - SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", False)) + SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", "False")) CONSUMER_SERVICE_NAME: str | None = os.getenv("CONSUMER_SERVICE_NAME") INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", "True")) SMTP_USERNAME: str = os.getenv("SMTP_USERNAME", "") From 5b30055e9a95abfcac6f4ed118407ed102891f5a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:08:16 +0200 Subject: [PATCH 103/123] Lint PLW1508 --- packages/syft/src/syft/serde/recursive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/serde/recursive.py b/packages/syft/src/syft/serde/recursive.py index 4c438975245..e2f196ec3dd 100644 --- a/packages/syft/src/syft/serde/recursive.py +++ b/packages/syft/src/syft/serde/recursive.py @@ -123,7 +123,7 @@ def skip_unregistered_class( """ search_unregistered_classes = ( - os.getenv("SYFT_SEARCH_MISSING_CANONICAL_NAME", False) == "true" + os.getenv("SYFT_SEARCH_MISSING_CANONICAL_NAME", "False") == "true" ) if not search_unregistered_classes: return False From 793001d7f5081c7b3ef9f3c6a1aba97ae426c6e3 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:09:45 +0200 Subject: [PATCH 104/123] Lint PLR1714 --- packages/syft/src/syft/server/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/server/run.py b/packages/syft/src/syft/server/run.py index 9fc4878a983..2ab90daa4db 100644 --- a/packages/syft/src/syft/server/run.py +++ b/packages/syft/src/syft/server/run.py @@ -9,7 +9,7 @@ def str_to_bool(bool_str: str | None) -> bool: result = False bool_str = str(bool_str).lower() - if bool_str == "true" or bool_str == "1": + if bool_str in ("true", "1"): result = True return result From 1605e53b6b966990c72a92c5dedff8754daf23db Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:10:37 +0200 Subject: [PATCH 105/123] Lint PLW1508 --- packages/syft/src/syft/server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/server/server.py b/packages/syft/src/syft/server/server.py index e386dacb7a6..57097ff4471 100644 --- a/packages/syft/src/syft/server/server.py +++ b/packages/syft/src/syft/server/server.py @@ -471,7 +471,7 @@ def init_blob_storage(self, config: BlobStorageConfig | None = None) -> None: ) config_ = OnDiskBlobStorageConfig( client_config=client_config, - min_blob_size=os.getenv("MIN_SIZE_BLOB_STORAGE_MB", 16), + min_blob_size=os.getenv("MIN_SIZE_BLOB_STORAGE_MB", "16"), ) else: config_ = config From db7eec3d79d094787e8a9897691b771d6e609974 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:14:22 +0200 Subject: [PATCH 106/123] Lint PLR1714 --- packages/syft/src/syft/service/api/api_service.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/syft/src/syft/service/api/api_service.py b/packages/syft/src/syft/service/api/api_service.py index 87051d9b4b4..6f4aac705bc 100644 --- a/packages/syft/src/syft/service/api/api_service.py +++ b/packages/syft/src/syft/service/api/api_service.py @@ -414,11 +414,7 @@ def _call_in_jobs( start = time.time() # TODO: what can we do here????? - while ( - job is None - or job.status == JobStatus.PROCESSING - or job.status == JobStatus.CREATED - ): + while (job is None or job.status in {JobStatus.PROCESSING, JobStatus.CREATED}: job = job_service.get(context, job_id) time.sleep(0.1) if (time.time() - custom_endpoint.endpoint_timeout) > start: From 7a01655931e0c2caeb15b5954b74f3ef45398856 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:16:26 +0200 Subject: [PATCH 107/123] Lint PLR1714 --- packages/syft/src/syft/service/code/user_code_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 4d200bdbf22..e6466f31dc2 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -653,8 +653,7 @@ def has_code_permission( self, code_item: UserCode, context: AuthedServiceContext ) -> SyftSuccess | SyftError: if not ( - context.credentials == context.server.verify_key - or context.credentials == code_item.user_verify_key + context.credentials in {context.server.verify_key, code_item.user_verify_key} ): return SyftError( message=f"Code Execution Permission: {context.credentials} denied" From dc65c96d1a90a6d8cf7ab16a5c9a17878d66da24 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:17:56 +0200 Subject: [PATCH 108/123] Lint --- packages/syft/src/syft/server/server.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/server/server.py b/packages/syft/src/syft/server/server.py index 57097ff4471..33d5c8a8901 100644 --- a/packages/syft/src/syft/server/server.py +++ b/packages/syft/src/syft/server/server.py @@ -1110,9 +1110,8 @@ def handle_api_call_with_unsigned_result( return SyftError( message=f"You sent a {type(api_call)}. This server requires SignedSyftAPICall." ) - else: - if not api_call.is_valid: - return SyftError(message="Your message signature is invalid") + elif not api_call.is_valid: + return SyftError(message="Your message signature is invalid") if api_call.message.server_uid != self.id and check_call_location: return self.forward_message(api_call=api_call) From 91ad819fefcdffe77cace4c1f6a8e219b7289b75 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:19:19 +0200 Subject: [PATCH 109/123] Lint --- packages/syft/src/syft/service/network/server_peer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/network/server_peer.py b/packages/syft/src/syft/service/network/server_peer.py index 941396820d5..0230eaf49a4 100644 --- a/packages/syft/src/syft/service/network/server_peer.py +++ b/packages/syft/src/syft/service/network/server_peer.py @@ -126,9 +126,8 @@ def pick_highest_priority_route(self, oldest: bool = True) -> ServerRoute: if oldest: if route.priority < highest_priority_route.priority: highest_priority_route = route - else: - if route.priority > highest_priority_route.priority: - highest_priority_route = route + elif route.priority > highest_priority_route.priority: + highest_priority_route = route return highest_priority_route def update_route(self, route: ServerRoute) -> None: From 05405f530c620f6230cbf01a85a55d029ad98f46 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:22:04 +0200 Subject: [PATCH 110/123] Lint --- .../syft/service/project/project_service.py | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/syft/src/syft/service/project/project_service.py b/packages/syft/src/syft/service/project/project_service.py index 0da9d043e18..ef33df496ef 100644 --- a/packages/syft/src/syft/service/project/project_service.py +++ b/packages/syft/src/syft/service/project/project_service.py @@ -106,20 +106,17 @@ def create_project( ) ) leader_server_peer = peer.ok() - else: + elif project.leader_server_route is not None: # for the leader server, as it does not have route information to itself # we rely on the data scientist to provide the route - # the route is then validated by the leader - if project.leader_server_route is not None: - leader_server_peer = ( - project.leader_server_route.validate_with_context( - context=context - ) - ) - else: - return SyftError( - message=f"project {project}'s leader_server_route is None" - ) + # the route is then validated by the leader + leader_server_peer = project.leader_server_route.validate_with_context( + context=context + ) + else: + return SyftError( + message=f"project {project}'s leader_server_route is None" + ) project_obj.leader_server_peer = leader_server_peer From c49c1e055cb99afa49599cf2c8145764d196a4dc Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:23:27 +0200 Subject: [PATCH 111/123] Lint --- packages/syft/src/syft/store/kv_document_store.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/store/kv_document_store.py b/packages/syft/src/syft/store/kv_document_store.py index 9935fe646dd..812a6023ef2 100644 --- a/packages/syft/src/syft/store/kv_document_store.py +++ b/packages/syft/src/syft/store/kv_document_store.py @@ -384,9 +384,8 @@ def _remove_keys( store_key.value in ck_col[pk_value_str] ): ck_col[pk_value_str].remove(store_key.value) - else: - if pk_value in ck_col and (store_key.value in ck_col[pk_value]): - ck_col[pk_value].remove(store_key.value) + elif pk_value in ck_col and (store_key.value in ck_col[pk_value]): + ck_col[pk_value].remove(store_key.value) self.searchable_keys[pk_key] = ck_col def _find_index_or_search_keys( From 478cba8ecaf67ddc9fc626015b609f91fbf430ff Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:26:09 +0200 Subject: [PATCH 112/123] Update api_service.py --- packages/syft/src/syft/service/api/api_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/api/api_service.py b/packages/syft/src/syft/service/api/api_service.py index 6f4aac705bc..8dea6d30bd7 100644 --- a/packages/syft/src/syft/service/api/api_service.py +++ b/packages/syft/src/syft/service/api/api_service.py @@ -414,7 +414,7 @@ def _call_in_jobs( start = time.time() # TODO: what can we do here????? - while (job is None or job.status in {JobStatus.PROCESSING, JobStatus.CREATED}: + while (job is None or job.status in {JobStatus.PROCESSING, JobStatus.CREATED}): job = job_service.get(context, job_id) time.sleep(0.1) if (time.time() - custom_endpoint.endpoint_timeout) > start: From f4fdbbebab0f88e86877b9bfb65d7021ee62e952 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:28:51 +0200 Subject: [PATCH 113/123] Lint --- packages/syft/src/syft/service/api/api_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/api/api_service.py b/packages/syft/src/syft/service/api/api_service.py index 8dea6d30bd7..7cb1a14dac3 100644 --- a/packages/syft/src/syft/service/api/api_service.py +++ b/packages/syft/src/syft/service/api/api_service.py @@ -414,7 +414,7 @@ def _call_in_jobs( start = time.time() # TODO: what can we do here????? - while (job is None or job.status in {JobStatus.PROCESSING, JobStatus.CREATED}): + while job is None or job.status in {JobStatus.PROCESSING, JobStatus.CREATED}: job = job_service.get(context, job_id) time.sleep(0.1) if (time.time() - custom_endpoint.endpoint_timeout) > start: From 7f43f1f01c16c9b8d5af81dccfdd5fd1ad2983e9 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:29:50 +0200 Subject: [PATCH 114/123] Lint --- packages/syft/src/syft/service/code/user_code_service.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index e6466f31dc2..ee697e6e8cd 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -652,9 +652,10 @@ def _call( def has_code_permission( self, code_item: UserCode, context: AuthedServiceContext ) -> SyftSuccess | SyftError: - if not ( - context.credentials in {context.server.verify_key, code_item.user_verify_key} - ): + if context.credentials not in { + context.server.verify_key, + code_item.user_verify_key, + }: return SyftError( message=f"Code Execution Permission: {context.credentials} denied" ) From a5d54e12a279aca52a5197ff6c191c323ffa5f0f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:30:52 +0200 Subject: [PATCH 115/123] Lint --- packages/syft/src/syft/service/project/project_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/project/project_service.py b/packages/syft/src/syft/service/project/project_service.py index ef33df496ef..49a38473f3b 100644 --- a/packages/syft/src/syft/service/project/project_service.py +++ b/packages/syft/src/syft/service/project/project_service.py @@ -109,7 +109,7 @@ def create_project( elif project.leader_server_route is not None: # for the leader server, as it does not have route information to itself # we rely on the data scientist to provide the route - # the route is then validated by the leader + # the route is then validated by the leader leader_server_peer = project.leader_server_route.validate_with_context( context=context ) From 33db3cc74f25e144436378bdc866a462b260702e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:34:15 +0200 Subject: [PATCH 116/123] Lint --- packages/grid/backend/grid/core/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 26ec2c94a1a..6ee42a02cef 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -138,7 +138,9 @@ def get_emails_enabled(self) -> Self: ) N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", "1")) SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") - SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", "False")) + SINGLE_CONTAINER_MODE: bool = str_to_bool( + os.getenv("SINGLE_CONTAINER_MODE", "False") + ) CONSUMER_SERVICE_NAME: str | None = os.getenv("CONSUMER_SERVICE_NAME") INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", "True")) SMTP_USERNAME: str = os.getenv("SMTP_USERNAME", "") From 44435d2f77e824524a7db16e2958693ce1c3ca44 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 18:35:40 +0200 Subject: [PATCH 117/123] revert --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b38a0b209c3..5e2e0170233 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -94,7 +94,7 @@ repos: types_or: [python, pyi, jupyter] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.0 + rev: v1.10.1 hooks: - id: mypy name: "mypy: syft-cli" @@ -119,7 +119,7 @@ repos: ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.0 + rev: v1.10.1 hooks: - id: mypy name: "mypy: grid" @@ -144,7 +144,7 @@ repos: ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.0 + rev: v1.10.1 hooks: - id: mypy name: "mypy: syft" From 0eb0be548a083790852c62170fe5cc267b8763ee Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 24 Jul 2024 19:13:38 +0200 Subject: [PATCH 118/123] Update ruff.toml --- ruff.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 0434081ba09..c7462c5751f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -16,7 +16,6 @@ select = [ "C4", # flake8-comprehensions # "PERF", # perflint "PL", # pylint - # "SIM", # flake8-simplify "UP", # pyupgrade ] ignore = [ From dcb89548023c841c9599e2ee618c33bd5530374a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 26 Jul 2024 22:23:34 +0200 Subject: [PATCH 119/123] Update rufff to version 0.5.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5e2e0170233..b9998e5bc33 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,7 +84,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.5.4" + rev: "v0.5.5" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] From 90cb53b802dcf3aea33cafb68f47672782900a9f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 31 Jul 2024 23:30:34 +0200 Subject: [PATCH 120/123] Lint PLW1509 --- packages/syft/src/syft/dev/prof.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/dev/prof.py b/packages/syft/src/syft/dev/prof.py index a6aeffc5780..3fd4c023ae1 100644 --- a/packages/syft/src/syft/dev/prof.py +++ b/packages/syft/src/syft/dev/prof.py @@ -33,7 +33,8 @@ def pyspy() -> None: # type: ignore "--pid", str(os.getpid()), ] - process = subprocess.Popen(command, preexec_fn=os.setsid) # nosec + # Use start_new_session=True to avoid using preexec_fn + process = subprocess.Popen(command, start_new_session=True) # nosec start_time = time.time() yield process From a1d9027544ae5f8a653d0c41c8eaab2a8acc4606 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 14 Aug 2024 21:46:46 +0200 Subject: [PATCH 121/123] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9998e5bc33..3ea4b25a5a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,7 +84,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.5.5" + rev: "v0.5.7" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] From 7017214921d445340d3aa74e53cdfaae0d0ae19e Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Mon, 19 Aug 2024 01:59:32 +0200 Subject: [PATCH 122/123] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26b6b60e97e..ca22813965a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,7 +84,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.5.7" + rev: "v0.6.1" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] From 786c4f1e875e5766441245a9bfbee9850b0021d6 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 28 Aug 2024 00:03:44 +0200 Subject: [PATCH 123/123] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca22813965a..2c1125f3489 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,7 +84,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.6.1" + rev: "v0.6.2" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes]