From f55d4bab6b8235a290486c5bdd5b463cf76ce8c1 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Tue, 12 Mar 2024 10:52:34 +0100 Subject: [PATCH 1/5] fix transform + privateattr warnings --- packages/syft/src/syft/service/dataset/dataset.py | 6 +++--- packages/syft/src/syft/types/syft_object.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 6a971bbfb1a..2b437b0b072 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -756,11 +756,11 @@ def create_and_store_twin(context: TransformContext) -> TransformContext: def infer_shape(context: TransformContext) -> TransformContext: - if context.output is not None and context.output["shape"] is None: + if context.output is None: + return SyftError(f"{context}'s output is None. No transformation happened") + if context.output["shape"] is None: if context.obj is not None and not _is_action_data_empty(context.obj.mock): context.output["shape"] = get_shape_or_len(context.obj.mock) - else: - print(f"{context}'s output is None. No transformation happened") return context diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 4084ae2020e..c8859ab5fca 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -579,9 +579,10 @@ def __post_init__(self) -> None: def _syft_set_validate_private_attrs_(self, **kwargs: Any) -> None: # Validate and set private attributes # https://github.com/pydantic/pydantic/issues/2105 + annotations = typing.get_type_hints(self.__class__) for attr, decl in self.__private_attributes__.items(): value = kwargs.get(attr, decl.get_default()) - var_annotation = self.__annotations__.get(attr) + var_annotation = annotations.get(attr) if value is not PydanticUndefined: if var_annotation is not None: # Otherwise validate value against the variable annotation From a0a0b5ea1c1e0f70f9bbbd58e1f608e674a84ab2 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Tue, 12 Mar 2024 11:07:46 +0100 Subject: [PATCH 2/5] change warnings to ValueError --- packages/syft/src/syft/service/dataset/dataset.py | 6 +++--- .../syft/src/syft/service/notification/notifications.py | 2 +- packages/syft/src/syft/service/policy/policy.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 2b437b0b072..daf92ecdbcb 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -757,7 +757,7 @@ def create_and_store_twin(context: TransformContext) -> TransformContext: def infer_shape(context: TransformContext) -> TransformContext: if context.output is None: - return SyftError(f"{context}'s output is None. No transformation happened") + raise ValueError(f"{context}'s output is None. No transformation happened") if context.output["shape"] is None: if context.obj is not None and not _is_action_data_empty(context.obj.mock): context.output["shape"] = get_shape_or_len(context.obj.mock) @@ -766,7 +766,7 @@ def infer_shape(context: TransformContext) -> TransformContext: def set_data_subjects(context: TransformContext) -> TransformContext | SyftError: if context.output is None: - return SyftError(f"{context}'s output is None. No transformation happened") + raise ValueError(f"{context}'s output is None. No transformation happened") if context.node is None: return SyftError( "f{context}'s node is None, please log in. No trasformation happened" @@ -796,7 +796,7 @@ def add_default_node_uid(context: TransformContext) -> TransformContext: if context.output["node_uid"] is None and context.node is not None: context.output["node_uid"] = context.node.id else: - print(f"{context}'s output is None. No transformation happened.") + raise ValueError(f"{context}'s output is None. No transformation happened") return context diff --git a/packages/syft/src/syft/service/notification/notifications.py b/packages/syft/src/syft/service/notification/notifications.py index e708993a7e0..6df1716ed4a 100644 --- a/packages/syft/src/syft/service/notification/notifications.py +++ b/packages/syft/src/syft/service/notification/notifications.py @@ -151,7 +151,7 @@ def add_msg_creation_time(context: TransformContext) -> TransformContext: if context.output is not None: context.output["created_at"] = DateTime.now() else: - print(f"{context}'s output is None. No transformation happened.") + raise ValueError(f"{context}'s output is None. No transformation happened") return context diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index 78b3b436765..841e2a7d049 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -572,7 +572,7 @@ def generate_unique_class_name(context: TransformContext) -> TransformContext: unique_name = f"{service_class_name}_{context.credentials}_{code_hash}" context.output["unique_name"] = unique_name else: - print(f"{context}'s output is None. No transformation happened.") + raise ValueError(f"{context}'s output is None. No transformation happened") return context @@ -696,7 +696,7 @@ def compile_code(context: TransformContext) -> TransformContext: + context.output["parsed_code"] ) else: - print(f"{context}'s output is None. No transformation happened.") + raise ValueError(f"{context}'s output is None. No transformation happened") return context From cdf6e8c7f0c4d6e9764d06b8a920a80d98f5a3c0 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Tue, 12 Mar 2024 11:12:47 +0100 Subject: [PATCH 3/5] fix syft_get_diffs --- packages/syft/src/syft/service/code/user_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 1c7b8d217cc..a4733016e23 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -117,7 +117,7 @@ class UserCodeStatusCollection(SyncableSyftObject): status_dict: dict[NodeIdentity, tuple[UserCodeStatus, str]] = {} user_code_link: LinkedObject - def get_diffs(self, ext_obj: Any) -> list[AttrDiff]: + def syft_get_diffs(self, ext_obj: Any) -> list[AttrDiff]: # relative from ...service.sync.diff_state import AttrDiff From 86fef340ddc3b69551297e4d232caa15f780118e Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Tue, 12 Mar 2024 14:57:09 +0100 Subject: [PATCH 4/5] fix get_type_hints --- packages/syft/src/syft/types/syft_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index c8859ab5fca..124794ee6d7 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -579,7 +579,7 @@ def __post_init__(self) -> None: def _syft_set_validate_private_attrs_(self, **kwargs: Any) -> None: # Validate and set private attributes # https://github.com/pydantic/pydantic/issues/2105 - annotations = typing.get_type_hints(self.__class__) + annotations = typing.get_type_hints(self.__class__, localns=locals()) for attr, decl in self.__private_attributes__.items(): value = kwargs.get(attr, decl.get_default()) var_annotation = annotations.get(attr) From 52d4ff58e3066c2cd3522e9d5ce042869b687faa Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Tue, 12 Mar 2024 17:11:51 +0100 Subject: [PATCH 5/5] fix get_type_hints for user-defined policies --- packages/syft/src/syft/service/action/action_object.py | 1 + packages/syft/src/syft/service/policy/policy.py | 5 +++++ packages/syft/src/syft/types/syft_object.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 451282e8f60..48191180b9b 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -230,6 +230,7 @@ class ActionObjectPointer: "__repr_str__", # pydantic "__repr_args__", # pydantic "__post_init__", # syft + "__validate_private_attrs__", # syft "id", # syft "to_mongo", # syft 🟡 TODO 23: Add composeable / inheritable object passthrough attrs "__attr_searchable__", # syft diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index 841e2a7d049..d0f8b2f7ce2 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -441,11 +441,16 @@ def apply_output( class UserOutputPolicy(OutputPolicy): __canonical_name__ = "UserOutputPolicy" + + # Do not validate private attributes of user-defined policies, User annotations can + # contain any type and throw a NameError when resolving. + __validate_private_attrs__ = False pass class UserInputPolicy(InputPolicy): __canonical_name__ = "UserInputPolicy" + __validate_private_attrs__ = False pass diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 124794ee6d7..3115c7d4da2 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -419,6 +419,7 @@ def make_id(cls, values: Any) -> Any: __attr_custom_repr__: ClassVar[list[str] | None] = ( None # show these in html repr of an object ) + __validate_private_attrs__: ClassVar[bool] = True def __syft_get_funcs__(self) -> list[tuple[str, Signature]]: funcs = print_type_cache[type(self)] @@ -577,6 +578,8 @@ def __post_init__(self) -> None: pass def _syft_set_validate_private_attrs_(self, **kwargs: Any) -> None: + if not self.__validate_private_attrs__: + return # Validate and set private attributes # https://github.com/pydantic/pydantic/issues/2105 annotations = typing.get_type_hints(self.__class__, localns=locals())