From 55a53e133dc93031f2f1566d772b46e2d296b5e5 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 8 May 2024 12:04:45 +0200 Subject: [PATCH 01/21] setting to enable eager execution --- packages/syft/src/syft/client/api.py | 11 ++ packages/syft/src/syft/client/client.py | 15 +- packages/syft/src/syft/node/node.py | 2 + .../src/syft/protocol/protocol_version.json | 182 +----------------- .../src/syft/service/action/action_object.py | 52 +++-- .../syft/service/metadata/node_metadata.py | 2 + .../src/syft/service/settings/settings.py | 2 + .../syft/service/settings/settings_service.py | 66 +++++-- packages/syft/tests/syft/api_test.py | 2 +- packages/syft/tests/syft/eager_test.py | 12 +- tests/integration/network/gateway_test.py | 20 +- 11 files changed, 128 insertions(+), 238 deletions(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index d60c6460b4f..8f57ddbb3e9 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -38,6 +38,7 @@ from ..serde.signature import signature_remove_self from ..service.context import AuthedServiceContext from ..service.context import ChangeContext +from ..service.metadata.node_metadata import NodeMetadataJSON from ..service.response import SyftAttributeError from ..service.response import SyftError from ..service.response import SyftSuccess @@ -227,6 +228,7 @@ class RemoteFunction(SyftObject): node_uid: UID signature: Signature + refresh_api_callback: Callable | None path: str make_call: Callable pre_kwargs: dict[str, Any] | None = None @@ -289,6 +291,13 @@ def function_call( return result = self.make_call(api_call=api_call, cache_result=cache_result) + # TODO: annotate this on the service method decorator + API_CALLS_THAT_REQUIRE_REFRESH = ["settings.enable_eager_execution"] + + if path in API_CALLS_THAT_REQUIRE_REFRESH: + if self.refresh_api_callback is not None: + self.refresh_api_callback() + result, _ = migrate_args_and_kwargs( [result], kwargs={}, to_latest_protocol=True ) @@ -490,6 +499,7 @@ def generate_remote_function( custom_function = bool(path == "api.call_in_jobs") remote_function = RemoteFunction( node_uid=node_uid, + refresh_api_callback=api.refresh_api_callback, signature=signature, path=path, make_call=make_call, @@ -763,6 +773,7 @@ class SyftAPI(SyftObject): refresh_api_callback: Callable | None = None __user_role: ServiceRole = ServiceRole.NONE communication_protocol: PROTOCOL_TYPE + metadata: NodeMetadataJSON | None = None # def __post_init__(self) -> None: # pass diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 9438294a6c0..04c0b0f15f7 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -242,7 +242,10 @@ def get_node_metadata( return NodeMetadataJSON(**metadata_json) def get_api( - self, credentials: SyftSigningKey, communication_protocol: int + self, + credentials: SyftSigningKey, + communication_protocol: int, + metadata: NodeMetadataJSON | None, ) -> SyftAPI: params = { "verify_key": str(credentials.verify_key), @@ -265,6 +268,7 @@ def get_api( obj.connection = self obj.signing_key = credentials obj.communication_protocol = communication_protocol + obj.metadata = metadata if self.proxy_target_uid: obj.node_uid = self.proxy_target_uid return cast(SyftAPI, obj) @@ -379,7 +383,10 @@ def to_blob_route(self, path: str, host: str | None = None) -> GridURL: return GridURL(port=8333).with_path(path) def get_api( - self, credentials: SyftSigningKey, communication_protocol: int + self, + credentials: SyftSigningKey, + communication_protocol: int, + metadata: NodeMetadataJSON | None, ) -> SyftAPI: # todo: its a bit odd to identify a user by its verify key maybe? if self.proxy_target_uid: @@ -401,6 +408,7 @@ def get_api( obj.connection = self obj.signing_key = credentials obj.communication_protocol = communication_protocol + obj.metadata = metadata if self.proxy_target_uid: obj.node_uid = self.proxy_target_uid return obj @@ -942,7 +950,9 @@ def _fetch_api(self, credentials: SyftSigningKey) -> SyftAPI: _api: SyftAPI = self.connection.get_api( credentials=credentials, communication_protocol=self.communication_protocol, + metadata=self.metadata, ) + self._fetch_node_metadata(self.credentials) def refresh_callback() -> SyftAPI: return self._fetch_api(self.credentials) @@ -958,6 +968,7 @@ def refresh_callback() -> SyftAPI: api=_api, ) self._api = _api + self._api.metadata = self.metadata return _api diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index ad0b6f7cd6e..137d636c231 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -988,6 +988,7 @@ def metadata(self) -> NodeMetadataV3: organization = settings_data.organization description = settings_data.description show_warnings = settings_data.show_warnings + eager_execution_enabled = settings_data.eager_execution_enabled node_type = self.node_type.value if self.node_type else "" node_side_type = self.node_side_type.value if self.node_side_type else "" @@ -1003,6 +1004,7 @@ def metadata(self) -> NodeMetadataV3: node_type=node_type, node_side_type=node_side_type, show_warnings=show_warnings, + eager_execution_enabled=eager_execution_enabled, ) @property diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index ca4d715a57b..9abc3f3d8eb 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -12,186 +12,6 @@ "release_name": "0.8.6.json" }, "dev": { - "object_versions": { - "NodeSettingsUpdate": { - "3": { - "version": 3, - "hash": "0f812fdd5aecc3e3aa1a7c953bbf7f8d8b03a77c5cdbb37e981fa91c8134c9f4", - "action": "add" - } - }, - "NodeSettings": { - "4": { - "version": 4, - "hash": "318e578f8a9af213a6af0cc2c567b62196b0ff81769d808afff4dd1eb7c372b8", - "action": "add" - } - }, - "EnclaveMetadata": { - "2": { - "version": 2, - "hash": "6dcc26695abc6a9ecd9d7d1e6507a9f1a92cc5ccd10987e92419bf984245f9a1", - "action": "remove" - }, - "3": { - "version": 3, - "hash": "d2f23411927c68e2307a84d180ad053b3e4ba12d74aba64d34dac224c90e815d", - "action": "add" - } - }, - "CustomEndpointActionObject": { - "1": { - "version": 1, - "hash": "642facc6cafbaad4de030a33cd619bd68ac31a32b0db07ddc1c1d5d7f914503e", - "action": "add" - } - }, - "JobItem": { - "4": { - "version": 4, - "hash": "6a7cc7c2bb4dd234c1508b0af4d3b403cd3b7b427578a775bf80dc36891923ed", - "action": "remove" - }, - "5": { - "version": 5, - "hash": "82ee08442b09797ed7a3710c31de633bb308b1d2215f51b58a3e01a4c201055d", - "action": "add" - } - }, - "ExecutionOutput": { - "1": { - "version": 1, - "hash": "c2337099eba14767ead75fcc1b1fa265c1898461ede0b5e7758a0e8d11d1757d", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "854fe9df5bcbb5c7e5b7c467bac423cd98c32f93d6876fea7b8eb6c08f6596da", - "action": "add" - } - }, - "TwinAPIContextView": { - "1": { - "version": 1, - "hash": "7d368102d0b473009af3b8c46e0ea6d35893c5ebb172b373ad7d52553c12c9fa", - "action": "add" - } - }, - "CustomAPIView": { - "1": { - "version": 1, - "hash": "0b9afdd554f0b353c07256e2522342be1302b395d649f1cbabc555e5baecb150", - "action": "add" - } - }, - "CustomApiEndpoint": { - "1": { - "version": 1, - "hash": "13617f3dce60fa692421e0d9deda7ffd365ec02d4a062c18510b48457b6eba02", - "action": "add" - } - }, - "PrivateAPIEndpoint": { - "1": { - "version": 1, - "hash": "004ec19753263440e2896b4e35d7a6305322934512f473f37d54043af5726fe6", - "action": "add" - } - }, - "PublicAPIEndpoint": { - "1": { - "version": 1, - "hash": "5589b6bdd045ee9c45987dae78fd5a1124530a6c493e2328b304d9273b75177f", - "action": "add" - } - }, - "UpdateTwinAPIEndpoint": { - "1": { - "version": 1, - "hash": "6d8effd404f15d4378b1ff3382e0622b9e5a637d9db342d43cfec00fe29c649a", - "action": "add" - } - }, - "CreateTwinAPIEndpoint": { - "1": { - "version": 1, - "hash": "55e0a7b0ac428a6abb771ffcb925ee79cdd752a4b83058aa4b71fbef2a9fee63", - "action": "add" - } - }, - "TwinAPIEndpoint": { - "1": { - "version": 1, - "hash": "e538734d20be3b477e188eb91f66600c2e654bb32e34806ef24329e48238bf18", - "action": "add" - } - }, - "SyftLog": { - "3": { - "version": 3, - "hash": "8964d48238672e0e5d5db6b932cda4ee8eb77581949ab3f7a38a05b1efec13b7", - "action": "remove" - }, - "4": { - "version": 4, - "hash": "ad6ef18ccd87fced669f3824d27ab423aaf52574b0cd4f720687aeaba77524e5", - "action": "add" - } - }, - "SyncState": { - "1": { - "version": 1, - "hash": "a0616775ec8ef0629e2d91e0df9cc4237ea3674727eda1ce367f1897ee35767d", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "925f1b8ccd4b9d542700a111f9c4bdd28bfa55978d805ddb2fb3c108cc940d19", - "action": "add" - } - }, - "NodePeer": { - "3": { - "version": 3, - "hash": "dababb03d2463b6218ae22d55293a60580f5a14bebd0c664d71da104e2f0b835", - "action": "add" - } - }, - "AssociationRequestChange": { - "1": { - "version": 1, - "hash": "508550c43e9e3f30243813c23eb6eec20918923d7ba09498cddbcd7e8bfa4539", - "action": "add" - } - }, - "APIEndpointQueueItem": { - "1": { - "version": 1, - "hash": "d31b2edfb767401c810584baccd27e4f566181c3ef7706618a82eb25ae20ff6d", - "action": "add" - } - }, - "SyncStateItem": { - "1": { - "version": 1, - "hash": "4dbfa0813f5a3f7be0b36249ff2d67e395ad7c9e138c5a122fc7342b8dcc4b92", - "action": "remove" - } - }, - "VeilidConnection": { - "1": { - "version": 1, - "hash": "c1796e7b01c9eae0dbf59cfd5c2c2f0e7eba593e0cea615717246572b27aae4b", - "action": "remove" - } - }, - "BlobRetrievalByURL": { - "5": { - "version": 5, - "hash": "4934bf72bb10ac0a670c87ab735175088274e090819436563543473e64cf15e3", - "action": "add" - } - } - } + "object_versions": {} } } diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 3cfe5fe2617..db2d4a5e1cc 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -415,6 +415,7 @@ class PreHookContext(SyftBaseObject): def make_action_side_effect( context: PreHookContext, *args: Any, **kwargs: Any ) -> Result[Ok[tuple[PreHookContext, tuple[Any, ...], dict[str, Any]]], Err[str]]: + print("MAKING ACTioN") """Create a new action from context_op_name, and add it to the PreHookContext Parameters: @@ -794,11 +795,12 @@ def _save_to_blob_storage_(self, data: Any) -> SyftError | None: from ...types.blob_storage import CreateBlobStorageEntry if not isinstance(data, ActionDataEmpty): - if isinstance(data, BlobFile) and not data.uploaded: - api = APIRegistry.api_for( - self.syft_node_location, self.syft_client_verify_key - ) - data.upload_to_blobstorage_from_api(api) + if isinstance(data, BlobFile): + if not data.uploaded: + api = APIRegistry.api_for( + self.syft_node_location, self.syft_client_verify_key + ) + data._upload_to_blobstorage_from_api(api) else: serialized = serialize(data, to_bytes=True) size = sys.getsizeof(serialized) @@ -1480,27 +1482,35 @@ def __post_init__(self) -> None: if HOOK_ON_POINTERS not in self.syft_post_hooks__: self.syft_pre_hooks__[HOOK_ON_POINTERS] = [] - # this should be a list as orders matters - for side_effect in [make_action_side_effect]: - if side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: - self.syft_pre_hooks__[HOOK_ALWAYS].append(side_effect) + api = APIRegistry.api_for(self.syft_node_location, self.syft_client_verify_key) + allow_eager_execution = ( + api is not None + and api.metadata is not None + and api.metadata.eager_execution_enabled + ) - for side_effect in [send_action_side_effect]: - if side_effect not in self.syft_pre_hooks__[HOOK_ON_POINTERS]: - self.syft_pre_hooks__[HOOK_ON_POINTERS].append(side_effect) + if allow_eager_execution: + # this should be a list as orders matters + for side_effect in [make_action_side_effect]: + if side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: + self.syft_pre_hooks__[HOOK_ALWAYS].append(side_effect) - if trace_action_side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: - self.syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) + for side_effect in [send_action_side_effect]: + if side_effect not in self.syft_pre_hooks__[HOOK_ON_POINTERS]: + self.syft_pre_hooks__[HOOK_ON_POINTERS].append(side_effect) - if HOOK_ALWAYS not in self.syft_post_hooks__: - self.syft_post_hooks__[HOOK_ALWAYS] = [] + if trace_action_side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: + self.syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) - if HOOK_ON_POINTERS not in self.syft_post_hooks__: - self.syft_post_hooks__[HOOK_ON_POINTERS] = [] + if HOOK_ALWAYS not in self.syft_post_hooks__: + self.syft_post_hooks__[HOOK_ALWAYS] = [] + + if HOOK_ON_POINTERS not in self.syft_post_hooks__: + self.syft_post_hooks__[HOOK_ON_POINTERS] = [] - for side_effect in [propagate_node_uid]: - if side_effect not in self.syft_post_hooks__[HOOK_ALWAYS]: - self.syft_post_hooks__[HOOK_ALWAYS].append(side_effect) + for side_effect in [propagate_node_uid]: + if side_effect not in self.syft_post_hooks__[HOOK_ALWAYS]: + self.syft_post_hooks__[HOOK_ALWAYS].append(side_effect) if isinstance(self.syft_action_data_type, ActionObject): raise Exception("Nested ActionObjects", self.syft_action_data_repr_) diff --git a/packages/syft/src/syft/service/metadata/node_metadata.py b/packages/syft/src/syft/service/metadata/node_metadata.py index 746e3336cd5..d6462176fa8 100644 --- a/packages/syft/src/syft/service/metadata/node_metadata.py +++ b/packages/syft/src/syft/service/metadata/node_metadata.py @@ -76,6 +76,7 @@ class NodeMetadataV3(SyftObject): description: str = "Text" node_side_type: str show_warnings: bool + eager_execution_enabled: bool def check_version(self, client_version: str) -> bool: return check_version( @@ -98,6 +99,7 @@ class NodeMetadataJSON(BaseModel, StorableObjectType): organization: str = "OpenMined" description: str = "My cool domain" signup_enabled: bool = False + eager_execution_enabled: bool = False admin_email: str = "" node_side_type: str show_warnings: bool diff --git a/packages/syft/src/syft/service/settings/settings.py b/packages/syft/src/syft/service/settings/settings.py index da4eb428d2a..6eb80a0090f 100644 --- a/packages/syft/src/syft/service/settings/settings.py +++ b/packages/syft/src/syft/service/settings/settings.py @@ -45,6 +45,7 @@ class NodeSettingsUpdate(PartialSyftObject): signup_enabled: bool admin_email: str association_request_auto_approval: bool + eager_execution_enabled: bool @serializable() @@ -68,6 +69,7 @@ class NodeSettings(SyftObject): description: str = "Text" node_type: NodeType = NodeType.DOMAIN signup_enabled: bool + eager_execution_enabled: bool = False admin_email: str node_side_type: NodeSideType = NodeSideType.HIGH_SIDE show_warnings: bool diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index 30e8242ceb9..73f2fded9c2 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -65,6 +65,20 @@ def set( def update( self, context: AuthedServiceContext, settings: NodeSettingsUpdate ) -> Result[SyftSuccess, SyftError]: + res = self._update(context, settings) + if res.is_ok(): + return SyftSuccess( + message=( + "Settings updated successfully. " + + "You must call .refresh() to sync your client with the changes." + ) + ) + else: + return SyftError(message=res.err()) + + def _update( + self, context: AuthedServiceContext, settings: NodeSettingsUpdate + ) -> Result[Ok, Err]: result = self.stash.get_all(context.credentials) if result.is_ok(): current_settings = result.ok() @@ -73,19 +87,11 @@ def update( update=settings.to_dict(exclude_empty=True) ) update_result = self.stash.update(context.credentials, new_settings) - if update_result.is_ok(): - return SyftSuccess( - message=( - "Settings updated successfully. " - + "You must call .refresh() to sync your client with the changes." - ) - ) - else: - return SyftError(message=update_result.err()) + return update_result else: - return SyftError(message="No settings found") + return Err(value="No settings found") else: - return SyftError(message=result.err()) + return result @service_method( path="settings.enable_notifications", @@ -124,20 +130,18 @@ def disable_notifications( return notifier_service.turn_off(context=context) @service_method( - path="settings.allow_guest_signup", - name="allow_guest_signup", + path="settings.enable_guest_signup", + name="enable_guest_signup", warning=HighSideCRUDWarning(confirmation=True), ) - def allow_guest_signup( + def enable_guest_signup( self, context: AuthedServiceContext, enable: bool ) -> SyftSuccess | SyftError: """Enable/Disable Registration for Data Scientist or Guest Users.""" flags.CAN_REGISTER = enable - method = context.node.get_service_method(SettingsService.update) settings = NodeSettingsUpdate(signup_enabled=enable) - - result = method(context=context, settings=settings) + result = self._update(context=context, settings=settings) if result.is_err(): return SyftError(message=f"Failed to update settings: {result.err()}") @@ -146,14 +150,34 @@ def allow_guest_signup( return SyftSuccess(message=f"Registration feature successfully {message}") @service_method( - path="settings.allow_association_request_auto_approval", - name="allow_association_request_auto_approval", + path="settings.enable_eager_execution", + name="enable_eager_execution", + warning=HighSideCRUDWarning(confirmation=True), + ) + def enable_eager_execution( + self, context: AuthedServiceContext, enable: bool + ) -> SyftSuccess | SyftError: + """Enable/Disable Registration for Data Scientist or Guest Users.""" + flags.CAN_REGISTER = enable + settings = NodeSettingsUpdate(eager_execution_enabled=enable) + + result = self._update(context=context, settings=settings) + + if result.is_err(): + return SyftError(message=f"Failed to update settings: {result.err()}") + + message = "enabled" if enable else "disabled" + return SyftSuccess(message=f"Eager execution {message}") + + @service_method( + path="settings.enable_association_request_auto_approval", + name="enable_association_request_auto_approval", ) - def allow_association_request_auto_approval( + def enable_association_request_auto_approval( self, context: AuthedServiceContext, enable: bool ) -> SyftSuccess | SyftError: new_settings = NodeSettingsUpdate(association_request_auto_approval=enable) - result = self.update(context, settings=new_settings) + result = self.update(context, settins=new_settings) if isinstance(result, SyftError): return result diff --git a/packages/syft/tests/syft/api_test.py b/packages/syft/tests/syft/api_test.py index 180060caffb..3c1ddd25072 100644 --- a/packages/syft/tests/syft/api_test.py +++ b/packages/syft/tests/syft/api_test.py @@ -42,7 +42,7 @@ def my_func(x): def test_api_cache_invalidation_login(root_verify_key, worker): guest_client = worker.guest_client - worker.root_client.settings.allow_guest_signup(enable=True) + worker.root_client.settings.enable_guest_signup(enable=True) assert guest_client.register( name="q", email="a@b.org", password="aaa", password_verify="aaa" ) diff --git a/packages/syft/tests/syft/eager_test.py b/packages/syft/tests/syft/eager_test.py index fcfb10d3bdb..3565b3ff7e0 100644 --- a/packages/syft/tests/syft/eager_test.py +++ b/packages/syft/tests/syft/eager_test.py @@ -12,6 +12,7 @@ def test_eager_permissions(worker, guest_client): root_domain_client = worker.root_client + assert root_domain_client.settings.enable_eager_execution(enable=True) input_obj = TwinObject( private_obj=np.array([[3, 3, 3], [3, 3, 3]]), mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), @@ -36,6 +37,7 @@ def test_eager_permissions(worker, guest_client): def test_plan(worker): root_domain_client = worker.root_client guest_client = worker.guest_client + assert root_domain_client.settings.enable_eager_execution(enable=True) @planify def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 @@ -77,6 +79,7 @@ def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 def test_plan_with_function_call(worker, guest_client): root_domain_client = worker.root_client guest_client = worker.guest_client + assert root_domain_client.settings.enable_eager_execution(enable=True) @planify def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 @@ -98,12 +101,13 @@ def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 def test_plan_with_object_instantiation(worker, guest_client): + root_domain_client = worker.root_client + assert root_domain_client.settings.enable_eager_execution(enable=True) + @planify def my_plan(x=np.array([1, 2, 3, 4, 5, 6])): # noqa: B008 return x + 1 - root_domain_client = worker.root_client - plan_ptr = my_plan.send(guest_client) input_obj = TwinObject( @@ -123,6 +127,7 @@ def my_plan(x=np.array([1, 2, 3, 4, 5, 6])): # noqa: B008 def test_setattribute(worker, guest_client): root_domain_client = worker.root_client + assert root_domain_client.settings.enable_eager_execution(enable=True) private_data, mock_data = ( np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), @@ -160,6 +165,7 @@ def test_setattribute(worker, guest_client): def test_getattribute(worker, guest_client): root_domain_client = worker.root_client + assert root_domain_client.settings.enable_eager_execution(enable=True) obj = TwinObject( private_obj=np.array([[1, 2, 3], [4, 5, 6]]), mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), @@ -176,6 +182,7 @@ def test_getattribute(worker, guest_client): def test_eager_method(worker, guest_client): root_domain_client = worker.root_client + assert root_domain_client.settings.enable_eager_execution(enable=True) obj = TwinObject( private_obj=np.array([[1, 2, 3], [4, 5, 6]]), @@ -197,6 +204,7 @@ def test_eager_method(worker, guest_client): def test_eager_dunder_method(worker, guest_client): root_domain_client = worker.root_client + assert root_domain_client.settings.enable_eager_execution(enable=True) obj = TwinObject( private_obj=np.array([[1, 2, 3], [4, 5, 6]]), diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index 25a8282096e..725a4500063 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -173,7 +173,7 @@ def test_dataset_search(set_env_var, gateway_port: int, domain_1_port: int) -> N port=domain_1_port, email="info@openmined.org", password="changethis" ) - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connect the domain to the gateway @@ -247,7 +247,7 @@ def test_domain_gateway_user_code( ) assert isinstance(user_create_res, SyftSuccess) - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # the domain client connects to the gateway @@ -300,7 +300,7 @@ def test_deleting_peers(set_env_var, domain_1_port: int, gateway_port: int) -> N ) # Enable automatic acceptance of association requests - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -365,7 +365,7 @@ def test_add_route(set_env_var, gateway_port: int, domain_1_port: int) -> None: assert isinstance(_remove_existing_peers(gateway_client), SyftSuccess) # Enable automatic acceptance of association requests - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -445,7 +445,7 @@ def test_delete_route(set_env_var, gateway_port: int, domain_1_port: int) -> Non ) # Enable automatic acceptance of association requests - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -500,7 +500,7 @@ def test_add_route_on_peer(set_env_var, gateway_port: int, domain_1_port: int) - assert isinstance(_remove_existing_peers(gateway_client), SyftSuccess) # Enable automatic acceptance of association requests - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -577,7 +577,7 @@ def test_delete_route_on_peer( ) # Enable automatic acceptance of association requests - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -644,7 +644,7 @@ def test_update_route_priority( assert isinstance(_remove_existing_peers(gateway_client), SyftSuccess) # Enable automatic acceptance of association requests - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -715,7 +715,7 @@ def test_update_route_priority_on_peer( assert isinstance(_remove_existing_peers(gateway_client), SyftSuccess) # Enable automatic acceptance of association requests - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -781,7 +781,7 @@ def test_dataset_stream(set_env_var, gateway_port: int, domain_1_port: int) -> N port=domain_1_port, email="info@openmined.org", password="changethis" ) - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connect the domain to the gateway From 6423ce24d3ae0d7ed136195b89b84e549eb1a7e1 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Mon, 27 May 2024 11:48:50 +0200 Subject: [PATCH 02/21] fix: remove print --- packages/syft/src/syft/service/action/action_object.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index db2d4a5e1cc..9e7f3248b1f 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -415,7 +415,6 @@ class PreHookContext(SyftBaseObject): def make_action_side_effect( context: PreHookContext, *args: Any, **kwargs: Any ) -> Result[Ok[tuple[PreHookContext, tuple[Any, ...], dict[str, Any]]], Err[str]]: - print("MAKING ACTioN") """Create a new action from context_op_name, and add it to the PreHookContext Parameters: From 4b573a1b61f191e4977552fbd99f8f7dcaf2b8d8 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Mon, 27 May 2024 11:49:24 +0200 Subject: [PATCH 03/21] fix: some allow_ -> enable_ renaming --- notebooks/api/0.8/00-load-data.ipynb | 2 +- notebooks/api/0.8/07-domain-register-control-flow.ipynb | 6 +++--- notebooks/tutorials/data-owner/02-account-management.ipynb | 4 ++-- notebooks/tutorials/model-auditing/colab/01-user-log.ipynb | 2 +- tests/integration/local/gateway_local_test.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/notebooks/api/0.8/00-load-data.ipynb b/notebooks/api/0.8/00-load-data.ipynb index c61fab17f41..d1cedc7ca57 100644 --- a/notebooks/api/0.8/00-load-data.ipynb +++ b/notebooks/api/0.8/00-load-data.ipynb @@ -662,7 +662,7 @@ "### Create a new Data Scientist account on the Domain Server\n", "\n", "Signup is disabled by default.\n", - "An Admin/DO can enable it by `domain_client.settings.allow_guest_signup(enable=True)`\n", + "An Admin/DO can enable it by `domain_client.settings.enable_guest_signup(enable=True)`\n", "\n", "Refer to notebook [07-domain-register-control-flow](./07-domain-register-control-flow.ipynb) for more information." ] diff --git a/notebooks/api/0.8/07-domain-register-control-flow.ipynb b/notebooks/api/0.8/07-domain-register-control-flow.ipynb index 5bd493a47c9..1c8721fa84d 100644 --- a/notebooks/api/0.8/07-domain-register-control-flow.ipynb +++ b/notebooks/api/0.8/07-domain-register-control-flow.ipynb @@ -90,7 +90,7 @@ "# The assumed state of this test is a node with signup set to False\n", "# however if the tox task has set it to True you need to overwrite the setting\n", "# before running the tests\n", - "# root_client.settings.allow_guest_signup(enable=False)" + "# root_client.settings.enable_guest_signup(enable=False)" ] }, { @@ -184,7 +184,7 @@ "outputs": [], "source": [ "# Enable guest signups\n", - "root_client.settings.allow_guest_signup(enable=True)" + "root_client.settings.enable_guest_signup(enable=True)" ] }, { @@ -261,7 +261,7 @@ "outputs": [], "source": [ "# Refresh the root client\n", - "root_client.settings.allow_guest_signup(enable=False)" + "root_client.settings.enable_guest_signup(enable=False)" ] }, { diff --git a/notebooks/tutorials/data-owner/02-account-management.ipynb b/notebooks/tutorials/data-owner/02-account-management.ipynb index a4e64b74698..a38df5a3c35 100644 --- a/notebooks/tutorials/data-owner/02-account-management.ipynb +++ b/notebooks/tutorials/data-owner/02-account-management.ipynb @@ -379,7 +379,7 @@ "metadata": {}, "outputs": [], "source": [ - "client.settings.allow_guest_signup(enable=True)" + "client.settings.enable_guest_signup(enable=True)" ] }, { @@ -422,7 +422,7 @@ "metadata": {}, "outputs": [], "source": [ - "client.settings.allow_guest_signup(enable=False)" + "client.settings.enable_guest_signup(enable=False)" ] }, { diff --git a/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb b/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb index 036c21f9ed6..5a414a80d33 100644 --- a/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb +++ b/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb @@ -167,7 +167,7 @@ "metadata": {}, "outputs": [], "source": [ - "mo_client.settings.allow_guest_signup(enable=True)" + "mo_client.settings.enable_guest_signup(enable=True)" ] }, { diff --git a/tests/integration/local/gateway_local_test.py b/tests/integration/local/gateway_local_test.py index a26e6ad35bb..5fd6f3f9f1f 100644 --- a/tests/integration/local/gateway_local_test.py +++ b/tests/integration/local/gateway_local_test.py @@ -141,7 +141,7 @@ def test_create_gateway( email="info@openmined.org", password="changethis", ) - res = gateway_client.settings.allow_association_request_auto_approval(enable=True) + res = gateway_client.settings.enable_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) domain_client: DomainClient = domain_webserver.login( From 9ad76ff592fc0132dd18cde9b8c0eefa38e4dbd2 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Mon, 27 May 2024 11:52:50 +0200 Subject: [PATCH 04/21] refactor: consistent naming for eager execution variable --- packages/syft/src/syft/service/action/action_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 9e7f3248b1f..76904bd4b51 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1482,13 +1482,13 @@ def __post_init__(self) -> None: self.syft_pre_hooks__[HOOK_ON_POINTERS] = [] api = APIRegistry.api_for(self.syft_node_location, self.syft_client_verify_key) - allow_eager_execution = ( + eager_execution_enabled = ( api is not None and api.metadata is not None and api.metadata.eager_execution_enabled ) - if allow_eager_execution: + if eager_execution_enabled: # this should be a list as orders matters for side_effect in [make_action_side_effect]: if side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: From 9874585bc9ff781b23b30079f7b176ee586d918b Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Mon, 27 May 2024 11:55:38 +0200 Subject: [PATCH 05/21] fix: typo in enable_association_request_auto_approval --- packages/syft/src/syft/service/settings/settings_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index 73f2fded9c2..fb2be494f05 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -177,7 +177,7 @@ def enable_association_request_auto_approval( self, context: AuthedServiceContext, enable: bool ) -> SyftSuccess | SyftError: new_settings = NodeSettingsUpdate(association_request_auto_approval=enable) - result = self.update(context, settins=new_settings) + result = self.update(context, settings=new_settings) if isinstance(result, SyftError): return result From 10c64295957dbc3157b1f83772327282b0b0b52d Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 11:08:46 +0200 Subject: [PATCH 06/21] fix: update a object versions --- packages/syft/src/syft/client/api.py | 37 ++++++- packages/syft/src/syft/client/client.py | 4 +- packages/syft/src/syft/node/node.py | 6 +- .../src/syft/protocol/protocol_version.json | 85 ++++++++++------ .../syft/service/metadata/metadata_service.py | 4 +- .../syft/service/metadata/node_metadata.py | 46 +++++---- .../syft/service/network/network_service.py | 4 +- .../src/syft/service/network/node_peer.py | 4 +- .../syft/src/syft/service/project/project.py | 6 +- .../src/syft/service/settings/settings.py | 96 ++++--------------- 10 files changed, 155 insertions(+), 137 deletions(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index e4e64956cc8..c40677221e2 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -51,6 +51,7 @@ from ..types.identity import Identity from ..types.syft_object import SYFT_OBJECT_VERSION_1 from ..types.syft_object import SYFT_OBJECT_VERSION_2 +from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.syft_object import SyftBaseObject from ..types.syft_object import SyftMigrationRegistry from ..types.syft_object import SyftObject @@ -245,7 +246,7 @@ class RemoteFunction(SyftObject): node_uid: UID signature: Signature - refresh_api_callback: Callable | None + refresh_api_callback: Callable | None = None path: str make_call: Callable pre_kwargs: dict[str, Any] | None = None @@ -815,6 +816,38 @@ def result_needs_api_update(api_call_result: Any) -> bool: return False +@serializable( + attrs=[ + "endpoints", + "node_uid", + "node_name", + "lib_endpoints", + "communication_protocol", + ] +) +class SyftAPIV2(SyftObject): + # version + __canonical_name__ = "SyftAPI" + __version__ = SYFT_OBJECT_VERSION_2 + + # fields + connection: NodeConnection | None = None + node_uid: UID | None = None + node_name: str | None = None + endpoints: dict[str, APIEndpoint] + lib_endpoints: dict[str, LibEndpoint] | None = None + api_module: APIModule | None = None + libs: APIModule | None = None + signing_key: SyftSigningKey | None = None + # serde / storage rules + refresh_api_callback: Callable | None = None + __user_role: ServiceRole = ServiceRole.NONE + communication_protocol: PROTOCOL_TYPE + + # informs getattr does not have nasty side effects + __syft_allow_autocomplete__ = ["services"] + + @instrument @serializable( attrs=[ @@ -828,7 +861,7 @@ def result_needs_api_update(api_call_result: Any) -> bool: class SyftAPI(SyftObject): # version __canonical_name__ = "SyftAPI" - __version__ = SYFT_OBJECT_VERSION_2 + __version__ = SYFT_OBJECT_VERSION_3 # fields connection: NodeConnection | None = None diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index af0d17e0c5e..d10e39b56b8 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -37,8 +37,8 @@ from ..serde.serializable import serializable from ..serde.serialize import _serialize from ..service.context import NodeServiceContext +from ..service.metadata.node_metadata import NodeMetadata from ..service.metadata.node_metadata import NodeMetadataJSON -from ..service.metadata.node_metadata import NodeMetadataV3 from ..service.response import SyftError from ..service.response import SyftSuccess from ..service.user.user import UserCreate @@ -705,7 +705,7 @@ def exchange_route( return self.api.services.network.exchange_credentials_with( self_node_route=self_node_route, remote_node_route=remote_node_route, - remote_node_verify_key=client.metadata.to(NodeMetadataV3).verify_key, + remote_node_verify_key=client.metadata.to(NodeMetadata).verify_key, ) else: raise ValueError( diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 6d221059dec..3f916468ad4 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -65,7 +65,7 @@ from ..service.job.job_stash import JobType from ..service.log.log_service import LogService from ..service.metadata.metadata_service import MetadataService -from ..service.metadata.node_metadata import NodeMetadataV3 +from ..service.metadata.node_metadata import NodeMetadata from ..service.network.network_service import NetworkService from ..service.network.utils import PeerHealthCheckTask from ..service.notification.notification_service import NotificationService @@ -1009,7 +1009,7 @@ def settings(self) -> NodeSettings: return settings @property - def metadata(self) -> NodeMetadataV3: + def metadata(self) -> NodeMetadata: name = "" organization = "" description = "" @@ -1023,7 +1023,7 @@ def metadata(self) -> NodeMetadataV3: node_type = self.node_type.value if self.node_type else "" node_side_type = self.node_side_type.value if self.node_side_type else "" - return NodeMetadataV3( + return NodeMetadata( name=name, id=self.id, verify_key=self.verify_key, diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 7d9fff05dac..f9c57552ee7 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -13,28 +13,66 @@ }, "dev": { "object_versions": { - "NodeSettingsUpdate": { + "NodeMetadata": { + "5": { + "version": 5, + "hash": "70197b4725dbdea0560ed8388e4d20b76808bee988f3630c5f916ee8f48761f8", + "action": "add" + } + }, + "SyftAPI": { "3": { "version": 3, - "hash": "0f812fdd5aecc3e3aa1a7c953bbf7f8d8b03a77c5cdbb37e981fa91c8134c9f4", + "hash": "b1b9d131a4f204ef2d56dc91bab3b945d5581080565232ede864f32015c0882a", "action": "add" + } + }, + "HTMLObject": { + "1": { + "version": 1, + "hash": "010d9aaca95f3fdfc8d1f97d01c1bd66483da774a59275b310c08d6912f7f863", + "action": "add" + } + }, + "NodeSettingsUpdate": { + "2": { + "version": 2, + "hash": "e1dc9d2f30c4aae1f7359eb3fd44de5537788cd3c69be5f30c36fb019f07c261", + "action": "remove" }, "4": { "version": 4, "hash": "e9120fdf714498cbd2f3da727a4d6e18ea2040b2e8487ea38dff4900de2b14d0", "action": "add" + }, + "5": { + "version": 5, + "hash": "fd89638bb3d6dda9095905aab7ed2883f0b3dd5245900e8e141eec87921c2c9e", + "action": "add" } }, "NodeSettings": { - "4": { - "version": 4, - "hash": "318e578f8a9af213a6af0cc2c567b62196b0ff81769d808afff4dd1eb7c372b8", - "action": "add" + "3": { + "version": 3, + "hash": "2d5f6e79f074f75b5cfc2357eac7cf635b8f083421009a513240b4dbbd5a0fc1", + "action": "remove" }, "5": { "version": 5, "hash": "cde18eb23fdffcfba47bc0e85efdbba1d59f1f5d6baa9c9690e1af14b35eb74e", "action": "add" + }, + "6": { + "version": 6, + "hash": "986d201418035e59b12787dfaf60aa2af17817c1894ce42ab4b982ed73127403", + "action": "add" + } + }, + "BlobRetrievalByURL": { + "5": { + "version": 5, + "hash": "4934bf72bb10ac0a670c87ab735175088274e090819436563543473e64cf15e3", + "action": "add" } }, "EnclaveMetadata": { @@ -80,6 +118,13 @@ "action": "add" } }, + "CreateCustomImageChange": { + "3": { + "version": 3, + "hash": "e5f099940a7623f145f51f3e15b97a910a1d7fda1f67739420fed3035d1f2995", + "action": "add" + } + }, "TwinAPIContextView": { "1": { "version": 1, @@ -181,6 +226,13 @@ "action": "add" } }, + "NodeMetadataUpdate": { + "2": { + "version": 2, + "hash": "520ae8ffc0c057ffa827cb7b267a19fb6b92e3cf3c0a3666ac34e271b6dd0aed", + "action": "remove" + } + }, "SyncStateItem": { "1": { "version": 1, @@ -194,27 +246,6 @@ "hash": "c1796e7b01c9eae0dbf59cfd5c2c2f0e7eba593e0cea615717246572b27aae4b", "action": "remove" } - }, - "BlobRetrievalByURL": { - "5": { - "version": 5, - "hash": "4934bf72bb10ac0a670c87ab735175088274e090819436563543473e64cf15e3", - "action": "add" - } - }, - "CreateCustomImageChange": { - "3": { - "version": 3, - "hash": "e5f099940a7623f145f51f3e15b97a910a1d7fda1f67739420fed3035d1f2995", - "action": "add" - } - }, - "HTMLObject": { - "1": { - "version": 1, - "hash": "010d9aaca95f3fdfc8d1f97d01c1bd66483da774a59275b310c08d6912f7f863", - "action": "add" - } } } } diff --git a/packages/syft/src/syft/service/metadata/metadata_service.py b/packages/syft/src/syft/service/metadata/metadata_service.py index c292b139b33..a1e7eebb799 100644 --- a/packages/syft/src/syft/service/metadata/metadata_service.py +++ b/packages/syft/src/syft/service/metadata/metadata_service.py @@ -8,7 +8,7 @@ from ..service import AbstractService from ..service import service_method from ..user.user_roles import GUEST_ROLE_LEVEL -from .node_metadata import NodeMetadataV3 +from .node_metadata import NodeMetadata @instrument @@ -20,7 +20,7 @@ def __init__(self, store: DocumentStore) -> None: @service_method( path="metadata.get_metadata", name="get_metadata", roles=GUEST_ROLE_LEVEL ) - def get_metadata(self, context: AuthedServiceContext) -> NodeMetadataV3: + def get_metadata(self, context: AuthedServiceContext) -> NodeMetadata: return context.node.metadata # type: ignore # @service_method(path="metadata.get_admin", name="get_admin", roles=GUEST_ROLE_LEVEL) diff --git a/packages/syft/src/syft/service/metadata/node_metadata.py b/packages/syft/src/syft/service/metadata/node_metadata.py index d6462176fa8..de60b90a412 100644 --- a/packages/syft/src/syft/service/metadata/node_metadata.py +++ b/packages/syft/src/syft/service/metadata/node_metadata.py @@ -14,8 +14,8 @@ from ...node.credentials import SyftVerifyKey from ...protocol.data_protocol import get_data_protocol from ...serde.serializable import serializable -from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import SYFT_OBJECT_VERSION_4 +from ...types.syft_object import SYFT_OBJECT_VERSION_5 from ...types.syft_object import StorableObjectType from ...types.syft_object import SyftObject from ...types.transforms import convert_types @@ -44,24 +44,33 @@ def check_version( @serializable() -class NodeMetadataUpdate(SyftObject): - __canonical_name__ = "NodeMetadataUpdate" - __version__ = SYFT_OBJECT_VERSION_2 - - name: str | None = None - organization: str | None = None - description: str | None = None - on_board: bool | None = None - id: UID | None = None # type: ignore[assignment] - verify_key: SyftVerifyKey | None = None - highest_object_version: int | None = None - lowest_object_version: int | None = None - syft_version: str | None = None - admin_email: str | None = None +class NodeMetadata(SyftObject): + __canonical_name__ = "NodeMetadata" + __version__ = SYFT_OBJECT_VERSION_5 + + name: str + id: UID + verify_key: SyftVerifyKey + highest_version: int + lowest_version: int + syft_version: str + node_type: NodeType = NodeType.DOMAIN + organization: str = "OpenMined" + description: str = "Text" + node_side_type: str + show_warnings: bool + eager_execution_enabled: bool + + def check_version(self, client_version: str) -> bool: + return check_version( + client_version=client_version, + server_version=self.syft_version, + server_name=self.name, + ) @serializable() -class NodeMetadataV3(SyftObject): +class NodeMetadataV4(SyftObject): __canonical_name__ = "NodeMetadata" __version__ = SYFT_OBJECT_VERSION_4 @@ -76,7 +85,6 @@ class NodeMetadataV3(SyftObject): description: str = "Text" node_side_type: str show_warnings: bool - eager_execution_enabled: bool def check_version(self, client_version: str) -> bool: return check_version( @@ -121,7 +129,7 @@ def check_version(self, client_version: str) -> bool: ) -@transform(NodeMetadataV3, NodeMetadataJSON) +@transform(NodeMetadata, NodeMetadataJSON) def metadata_to_json() -> list[Callable]: return [ drop(["__canonical_name__"]), @@ -132,7 +140,7 @@ def metadata_to_json() -> list[Callable]: ] -@transform(NodeMetadataJSON, NodeMetadataV3) +@transform(NodeMetadataJSON, NodeMetadata) def json_to_metadata() -> list[Callable]: return [ drop(["metadata_version", "supported_protocols"]), diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 10074353146..410b996eeca 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -33,7 +33,7 @@ from ...util.util import prompt_warning_message from ..context import AuthedServiceContext from ..data_subject.data_subject import NamePartitionKey -from ..metadata.node_metadata import NodeMetadataV3 +from ..metadata.node_metadata import NodeMetadata from ..request.request import Request from ..request.request import RequestStatus from ..request.request import SubmitRequest @@ -953,7 +953,7 @@ def node_route_to_http_connection( return HTTPConnection(url=url, proxy_target_uid=obj.proxy_target_uid) -@transform(NodeMetadataV3, NodePeer) +@transform(NodeMetadata, NodePeer) def metadata_to_peer() -> list[Callable]: return [ keep( diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index 35292dd89dd..4a5447e293d 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -23,7 +23,7 @@ from ...types.transforms import TransformContext from ...types.uid import UID from ..context import NodeServiceContext -from ..metadata.node_metadata import NodeMetadataV3 +from ..metadata.node_metadata import NodeMetadata from .routes import HTTPNodeRoute from .routes import NodeRoute from .routes import NodeRouteType @@ -210,7 +210,7 @@ def from_client(client: SyftClient) -> "NodePeer": if not client.metadata: raise ValueError("Client has to have metadata first") - peer = client.metadata.to(NodeMetadataV3).to(NodePeer) + peer = client.metadata.to(NodeMetadata).to(NodePeer) route = connection_to_route(client.connection) peer.node_routes.append(route) return peer diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index d9b84ef9f15..981f7ff9192 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -24,7 +24,7 @@ from ...node.credentials import SyftVerifyKey from ...serde.serializable import serializable from ...serde.serialize import _serialize -from ...service.metadata.node_metadata import NodeMetadataV3 +from ...service.metadata.node_metadata import NodeMetadata from ...store.linked_obj import LinkedObject from ...types.datetime import DateTime from ...types.identity import Identity @@ -60,7 +60,7 @@ class EventAlreadyAddedException(SyftException): pass -@transform(NodeMetadataV3, NodeIdentity) +@transform(NodeMetadata, NodeIdentity) def metadata_to_node_identity() -> list[Callable]: return [rename("id", "node_id"), rename("name", "node_name")] @@ -1245,7 +1245,7 @@ def to_node_identity(val: SyftClient | NodeIdentity) -> NodeIdentity: if isinstance(val, NodeIdentity): return val elif isinstance(val, SyftClient) and val.metadata is not None: - metadata = val.metadata.to(NodeMetadataV3) + metadata = val.metadata.to(NodeMetadata) return metadata.to(NodeIdentity) else: raise SyftException( diff --git a/packages/syft/src/syft/service/settings/settings.py b/packages/syft/src/syft/service/settings/settings.py index 972d3dac36f..b7f31290720 100644 --- a/packages/syft/src/syft/service/settings/settings.py +++ b/packages/syft/src/syft/service/settings/settings.py @@ -10,9 +10,9 @@ from ...service.worker.utils import DEFAULT_WORKER_POOL_NAME from ...types.syft_migration import migrate from ...types.syft_object import PartialSyftObject -from ...types.syft_object import SYFT_OBJECT_VERSION_2 -from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...types.syft_object import SYFT_OBJECT_VERSION_4 +from ...types.syft_object import SYFT_OBJECT_VERSION_5 +from ...types.syft_object import SYFT_OBJECT_VERSION_6 from ...types.syft_object import SyftObject from ...types.transforms import drop from ...types.transforms import make_set_default @@ -25,24 +25,9 @@ @serializable() -class NodeSettingsUpdateV2(PartialSyftObject): +class NodeSettingsUpdateV4(PartialSyftObject): __canonical_name__ = "NodeSettingsUpdate" - __version__ = SYFT_OBJECT_VERSION_2 - - id: UID - name: str - organization: str - description: str - on_board: bool - signup_enabled: bool - admin_email: str - - -@serializable() -class NodeSettingsUpdateV3(PartialSyftObject): - __canonical_name__ = "NodeSettingsUpdate" - __version__ = SYFT_OBJECT_VERSION_3 - + __version__ = SYFT_OBJECT_VERSION_4 id: UID name: str organization: str @@ -51,13 +36,13 @@ class NodeSettingsUpdateV3(PartialSyftObject): signup_enabled: bool admin_email: str association_request_auto_approval: bool - eager_execution_enabled: bool + welcome_markdown: HTMLObject | MarkdownDescription @serializable() class NodeSettingsUpdate(PartialSyftObject): __canonical_name__ = "NodeSettingsUpdate" - __version__ = SYFT_OBJECT_VERSION_4 + __version__ = SYFT_OBJECT_VERSION_5 id: UID name: str organization: str @@ -67,12 +52,13 @@ class NodeSettingsUpdate(PartialSyftObject): admin_email: str association_request_auto_approval: bool welcome_markdown: HTMLObject | MarkdownDescription + eager_execution_enabled: bool = False @serializable() class NodeSettings(SyftObject): __canonical_name__ = "NodeSettings" - __version__ = 5 + __version__ = SYFT_OBJECT_VERSION_6 __repr_attrs__ = [ "name", "organization", @@ -94,6 +80,7 @@ class NodeSettings(SyftObject): node_side_type: NodeSideType = NodeSideType.HIGH_SIDE show_warnings: bool association_request_auto_approval: bool + eager_execution_enabled: bool = False default_worker_pool: str = DEFAULT_WORKER_POOL_NAME welcome_markdown: HTMLObject | MarkdownDescription = HTMLObject( text=DEFAULT_WELCOME_MSG @@ -118,9 +105,9 @@ def _repr_html_(self) -> Any: @serializable() -class NodeSettingsV4(SyftObject): +class NodeSettingsV5(SyftObject): __canonical_name__ = "NodeSettings" - __version__ = SYFT_OBJECT_VERSION_4 + __version__ = SYFT_OBJECT_VERSION_5 __repr_attrs__ = [ "name", "organization", @@ -138,72 +125,31 @@ class NodeSettingsV4(SyftObject): description: str = "Text" node_type: NodeType = NodeType.DOMAIN signup_enabled: bool - eager_execution_enabled: bool = False admin_email: str node_side_type: NodeSideType = NodeSideType.HIGH_SIDE show_warnings: bool association_request_auto_approval: bool default_worker_pool: str = DEFAULT_WORKER_POOL_NAME - - def _repr_html_(self) -> Any: - return f""" - -
-

Settings

-

Id: {self.id}

-

Name: {self.name}

-

Organization: {self.organization}

-

Deployed on: {self.deployed_on}

-

Signup enabled: {self.signup_enabled}

-

Admin email: {self.admin_email}

-
- - """ - - -@serializable() -class NodeSettingsV2(SyftObject): - __canonical_name__ = "NodeSettings" - __version__ = SYFT_OBJECT_VERSION_3 - __repr_attrs__ = [ - "name", - "organization", - "deployed_on", - "signup_enabled", - "admin_email", - ] - - id: UID - name: str = "Node" - deployed_on: str - organization: str = "OpenMined" - verify_key: SyftVerifyKey - on_board: bool = True - description: str = "Text" - node_type: NodeType = NodeType.DOMAIN - signup_enabled: bool - admin_email: str - node_side_type: NodeSideType = NodeSideType.HIGH_SIDE - show_warnings: bool + welcome_markdown: HTMLObject | MarkdownDescription = HTMLObject( + text=DEFAULT_WELCOME_MSG + ) -@migrate(NodeSettingsV2, NodeSettings) +@migrate(NodeSettingsV5, NodeSettings) def upgrade_node_settings() -> list[Callable]: - return [make_set_default("association_request_auto_approval", False)] + return [make_set_default("eager_execution_enabled", False)] -@migrate(NodeSettings, NodeSettingsV2) +@migrate(NodeSettings, NodeSettingsV5) def downgrade_node_settings() -> list[Callable]: - return [drop(["association_request_auto_approval"])] + return [drop(["eager_execution_enabled"])] -@migrate(NodeSettingsUpdateV2, NodeSettingsUpdate) +@migrate(NodeSettingsUpdateV4, NodeSettingsUpdate) def upgrade_node_settings_update() -> list[Callable]: return [] -@migrate(NodeSettings, NodeSettingsV2) +@migrate(NodeSettingsUpdate, NodeSettingsUpdateV4) def downgrade_node_settings_update() -> list[Callable]: - return [drop(["association_request_auto_approval"])] + return [drop(["eager_execution_enabled"])] From ad826bfbb1b51cdb2caf4c0ce79e641104b8ed76 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 11:14:13 +0200 Subject: [PATCH 07/21] fix: remove flag from eager_registration service fn --- packages/syft/src/syft/service/settings/settings_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index 4cb2ed3da2c..918cc9826ec 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -189,8 +189,7 @@ def enable_guest_signup( def enable_eager_execution( self, context: AuthedServiceContext, enable: bool ) -> SyftSuccess | SyftError: - """Enable/Disable Registration for Data Scientist or Guest Users.""" - flags.CAN_REGISTER = enable + """Enable/Disable eager execution.""" settings = NodeSettingsUpdate(eager_execution_enabled=enable) result = self._update(context=context, settings=settings) From 490f9d9fc60f08558961c0300282c5eaf2898805 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 17:49:35 +0200 Subject: [PATCH 08/21] chore: fix comment typo --- packages/syft/src/syft/service/action/action_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 76904bd4b51..e32b17d95be 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -868,7 +868,7 @@ def is_pointer(self) -> bool: @property def syft_lineage_id(self) -> LineageID: - """Compute the LineageID of the ActionObject, using the `id` and the `syft_history_hash` memebers""" + """Compute the LineageID of the ActionObject, using the `id` and the `syft_history_hash` members""" return LineageID(self.id, self.syft_history_hash) model_config = ConfigDict(validate_assignment=True) @@ -1644,7 +1644,7 @@ def _syft_attr_propagate_ids( result.syft_node_location = context.syft_node_location result.syft_client_verify_key = context.syft_client_verify_key - # Propogate Syft blob storage entry id + # Propagate Syft blob storage entry id object_attrs = [ "syft_blob_storage_entry_id", "syft_action_data_repr_", From f6d59f07717beae21db62edc8a3243ae2ccf8e0c Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 17:50:56 +0200 Subject: [PATCH 09/21] fix: now checks for HOOK_POINTERS in syft_pre_hook instead of syft_post_hook during __post_init__ --- packages/syft/src/syft/service/action/action_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index e32b17d95be..b3e8de11384 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1478,7 +1478,7 @@ def __post_init__(self) -> None: if HOOK_ALWAYS not in self.syft_pre_hooks__: self.syft_pre_hooks__[HOOK_ALWAYS] = [] - if HOOK_ON_POINTERS not in self.syft_post_hooks__: + if HOOK_ON_POINTERS not in self.syft_pre_hooks__: self.syft_pre_hooks__[HOOK_ON_POINTERS] = [] api = APIRegistry.api_for(self.syft_node_location, self.syft_client_verify_key) From 1600c9361528c533f2c5924b851eb25eef9af424 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 18:01:55 +0200 Subject: [PATCH 10/21] fix(action_object): moved post_hook init next to pre_hook --- .../syft/src/syft/service/action/action_object.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index b3e8de11384..12e5b44a22f 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1481,6 +1481,12 @@ def __post_init__(self) -> None: if HOOK_ON_POINTERS not in self.syft_pre_hooks__: self.syft_pre_hooks__[HOOK_ON_POINTERS] = [] + if HOOK_ALWAYS not in self.syft_post_hooks__: + self.syft_post_hooks__[HOOK_ALWAYS] = [] + + if HOOK_ON_POINTERS not in self.syft_post_hooks__: + self.syft_post_hooks__[HOOK_ON_POINTERS] = [] + api = APIRegistry.api_for(self.syft_node_location, self.syft_client_verify_key) eager_execution_enabled = ( api is not None @@ -1489,7 +1495,7 @@ def __post_init__(self) -> None: ) if eager_execution_enabled: - # this should be a list as orders matters + # pre_hooks | this should be a list as orders matters for side_effect in [make_action_side_effect]: if side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: self.syft_pre_hooks__[HOOK_ALWAYS].append(side_effect) @@ -1501,12 +1507,7 @@ def __post_init__(self) -> None: if trace_action_side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: self.syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) - if HOOK_ALWAYS not in self.syft_post_hooks__: - self.syft_post_hooks__[HOOK_ALWAYS] = [] - - if HOOK_ON_POINTERS not in self.syft_post_hooks__: - self.syft_post_hooks__[HOOK_ON_POINTERS] = [] - + # post_hooks | this should be a list as orders matters for side_effect in [propagate_node_uid]: if side_effect not in self.syft_post_hooks__[HOOK_ALWAYS]: self.syft_post_hooks__[HOOK_ALWAYS].append(side_effect) From 368416d3393a82d397893e43a4ea407d5577505f Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 19:03:32 +0200 Subject: [PATCH 11/21] fix: send and trace side effects only run on eager execution --- .../syft/src/syft/service/action/action_object.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 12e5b44a22f..f66c0d74964 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -428,8 +428,6 @@ def make_action_side_effect( - Ok[[Tuple[PreHookContext, Tuple[Any, ...], Dict[str, Any]]] on success - Err[str] on failure """ - # relative - try: action = context.obj.syft_make_action_with_self( op=context.op_name, @@ -1494,12 +1492,12 @@ def __post_init__(self) -> None: and api.metadata.eager_execution_enabled ) - if eager_execution_enabled: - # pre_hooks | this should be a list as orders matters - for side_effect in [make_action_side_effect]: - if side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: - self.syft_pre_hooks__[HOOK_ALWAYS].append(side_effect) + # pre_hooks | this should be a list as orders matters + for side_effect in [make_action_side_effect]: + if side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: + self.syft_pre_hooks__[HOOK_ALWAYS].append(side_effect) + if eager_execution_enabled: for side_effect in [send_action_side_effect]: if side_effect not in self.syft_pre_hooks__[HOOK_ON_POINTERS]: self.syft_pre_hooks__[HOOK_ON_POINTERS].append(side_effect) From 32d20582c74d766270317c0d36db86ecfd35aad4 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 19:25:09 +0200 Subject: [PATCH 12/21] fix: force eager execution for plans --- packages/syft/src/syft/service/action/plan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/syft/src/syft/service/action/plan.py b/packages/syft/src/syft/service/action/plan.py index 0bab10c0958..87730375473 100644 --- a/packages/syft/src/syft/service/action/plan.py +++ b/packages/syft/src/syft/service/action/plan.py @@ -66,6 +66,7 @@ def planify(func: Callable) -> ActionObject: ActionObject.add_trace_hook() worker = Worker.named(name="plan_building", reset=True, processes=0) client = worker.root_client + client.settings.enable_eager_execution(enable=True) if client is None: raise ValueError("Not able to get client for plan building") TraceResultRegistry.set_trace_result_for_current_thread(client=client) From 131e978a810a58bd788f840f70e5f728024f8a4e Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 21:10:19 +0200 Subject: [PATCH 13/21] tests: add eager execution flag to tests --- packages/syft/tests/syft/action_test.py | 1 + packages/syft/tests/syft/eager_test.py | 12 ++++++++++-- .../tests/syft/service/action/action_object_test.py | 4 +++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/syft/tests/syft/action_test.py b/packages/syft/tests/syft/action_test.py index a9b2adb1c97..c28f5c31615 100644 --- a/packages/syft/tests/syft/action_test.py +++ b/packages/syft/tests/syft/action_test.py @@ -14,6 +14,7 @@ def test_actionobject_method(worker): root_domain_client = worker.root_client + assert root_domain_client.settings.enable_eager_execution(enable=True) action_store = worker.get_service("actionservice").store obj = ActionObject.from_obj("abc") pointer = root_domain_client.api.services.action.set(obj) diff --git a/packages/syft/tests/syft/eager_test.py b/packages/syft/tests/syft/eager_test.py index 3565b3ff7e0..a597a79b2d6 100644 --- a/packages/syft/tests/syft/eager_test.py +++ b/packages/syft/tests/syft/eager_test.py @@ -13,6 +13,8 @@ def test_eager_permissions(worker, guest_client): root_domain_client = worker.root_client assert root_domain_client.settings.enable_eager_execution(enable=True) + guest_client = worker.guest_client + input_obj = TwinObject( private_obj=np.array([[3, 3, 3], [3, 3, 3]]), mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), @@ -36,8 +38,8 @@ def test_eager_permissions(worker, guest_client): def test_plan(worker): root_domain_client = worker.root_client - guest_client = worker.guest_client assert root_domain_client.settings.enable_eager_execution(enable=True) + guest_client = worker.guest_client @planify def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 @@ -78,8 +80,8 @@ def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 @currently_fail_on_python_3_12(raises=AttributeError) def test_plan_with_function_call(worker, guest_client): root_domain_client = worker.root_client - guest_client = worker.guest_client assert root_domain_client.settings.enable_eager_execution(enable=True) + guest_client = worker.guest_client @planify def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 @@ -103,6 +105,7 @@ def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 def test_plan_with_object_instantiation(worker, guest_client): root_domain_client = worker.root_client assert root_domain_client.settings.enable_eager_execution(enable=True) + guest_client = worker.guest_client @planify def my_plan(x=np.array([1, 2, 3, 4, 5, 6])): # noqa: B008 @@ -128,6 +131,7 @@ def my_plan(x=np.array([1, 2, 3, 4, 5, 6])): # noqa: B008 def test_setattribute(worker, guest_client): root_domain_client = worker.root_client assert root_domain_client.settings.enable_eager_execution(enable=True) + guest_client = worker.guest_client private_data, mock_data = ( np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), @@ -166,6 +170,8 @@ def test_setattribute(worker, guest_client): def test_getattribute(worker, guest_client): root_domain_client = worker.root_client assert root_domain_client.settings.enable_eager_execution(enable=True) + guest_client = worker.guest_client + obj = TwinObject( private_obj=np.array([[1, 2, 3], [4, 5, 6]]), mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), @@ -183,6 +189,7 @@ def test_getattribute(worker, guest_client): def test_eager_method(worker, guest_client): root_domain_client = worker.root_client assert root_domain_client.settings.enable_eager_execution(enable=True) + guest_client = worker.guest_client obj = TwinObject( private_obj=np.array([[1, 2, 3], [4, 5, 6]]), @@ -205,6 +212,7 @@ def test_eager_method(worker, guest_client): def test_eager_dunder_method(worker, guest_client): root_domain_client = worker.root_client assert root_domain_client.settings.enable_eager_execution(enable=True) + guest_client = worker.guest_client obj = TwinObject( private_obj=np.array([[1, 2, 3], [4, 5, 6]]), diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index fa8efab4eaf..05f8d4b0a92 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -566,6 +566,8 @@ def test_actionobject_syft_get_attr_context(): ) def test_actionobject_syft_execute_hooks(worker, testcase): client = worker.root_client + assert client.settings.enable_eager_execution(enable=True) + orig_obj, op, args, kwargs, expected = testcase obj = helper_make_action_obj(orig_obj) @@ -918,7 +920,7 @@ def test_actionobject_syft_getattr_int(orig_obj: int, worker, scenario): assert (3 >> obj) == (3 >> orig_obj) -def test_actionobject_syft_getattr_int_history(worker): +def test_actionobject_syft_getattr_int_history(): orig_obj = 5 obj1 = ActionObject.from_obj(orig_obj) obj2 = ActionObject.from_obj(orig_obj) From d0802d14391ce9f6875084fbd472fe0c629bc13c Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 21:12:06 +0200 Subject: [PATCH 14/21] fix(action_object): split pre/post hook functions for testing --- .../src/syft/service/action/action_object.py | 37 +++++++++++++++---- .../syft/service/action/action_object_test.py | 17 ++++++++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index f66c0d74964..cea9ca9a905 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1492,12 +1492,29 @@ def __post_init__(self) -> None: and api.metadata.eager_execution_enabled ) - # pre_hooks | this should be a list as orders matters + self._syft_add_pre_hooks__(eager_execution_enabled) + self._syft_add_post_hooks__(eager_execution_enabled) + + if isinstance(self.syft_action_data_type, ActionObject): + raise Exception("Nested ActionObjects", self.syft_action_data_repr_) + + self.syft_history_hash = hash(self.id) + + def _syft_add_pre_hooks__(self, eager_execution: bool): + """ + Add pre-hooks + + Args: + eager_execution: bool: If eager execution is enabled, hooks for + tracing and executing the action on remote are added. + """ + + # this should be a list as orders matters for side_effect in [make_action_side_effect]: if side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: self.syft_pre_hooks__[HOOK_ALWAYS].append(side_effect) - if eager_execution_enabled: + if eager_execution: for side_effect in [send_action_side_effect]: if side_effect not in self.syft_pre_hooks__[HOOK_ON_POINTERS]: self.syft_pre_hooks__[HOOK_ON_POINTERS].append(side_effect) @@ -1505,16 +1522,20 @@ def __post_init__(self) -> None: if trace_action_side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: self.syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) - # post_hooks | this should be a list as orders matters + def _syft_add_post_hooks__(self, eager_execution: bool): + """ + Add post-hooks + + Args: + eager_execution: bool: If eager execution is enabled, hooks for + tracing and executing the action on remote are added. + """ + if eager_execution: + # this should be a list as orders matters for side_effect in [propagate_node_uid]: if side_effect not in self.syft_post_hooks__[HOOK_ALWAYS]: self.syft_post_hooks__[HOOK_ALWAYS].append(side_effect) - if isinstance(self.syft_action_data_type, ActionObject): - raise Exception("Nested ActionObjects", self.syft_action_data_repr_) - - self.syft_history_hash = hash(self.id) - def _syft_run_pre_hooks__( self, context: PreHookContext, name: str, args: Any, kwargs: Any ) -> tuple[PreHookContext, tuple[Any, ...], dict[str, Any]]: diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index 05f8d4b0a92..fe13a558e91 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -160,12 +160,27 @@ def test_actionobject_hooks_init(orig_obj: Any): assert HOOK_ALWAYS in obj.syft_pre_hooks__ assert HOOK_ALWAYS in obj.syft_post_hooks__ + assert HOOK_ON_POINTERS in obj.syft_pre_hooks__ + assert HOOK_ON_POINTERS in obj.syft_post_hooks__ + + assert make_action_side_effect in obj.syft_pre_hooks__[HOOK_ALWAYS] + +def test_actionobject_add_pre_hooks(): + # Eager execution is disabled by default + obj = ActionObject.from_obj(1) + + assert make_action_side_effect in obj.syft_pre_hooks__[HOOK_ALWAYS] + assert send_action_side_effect not in obj.syft_pre_hooks__[HOOK_ON_POINTERS] + assert propagate_node_uid not in obj.syft_post_hooks__[HOOK_ALWAYS] + + # eager exec tests: + obj._syft_add_pre_hooks__(eager_execution=True) + obj._syft_add_post_hooks__(eager_execution=True) assert make_action_side_effect in obj.syft_pre_hooks__[HOOK_ALWAYS] assert send_action_side_effect in obj.syft_pre_hooks__[HOOK_ON_POINTERS] assert propagate_node_uid in obj.syft_post_hooks__[HOOK_ALWAYS] - @pytest.mark.parametrize( "orig_obj_op", [ From 0b773701940c12eb286e7b05679eca5cab7ac0c2 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 21:22:06 +0200 Subject: [PATCH 15/21] fix: revert rename from enable to allow for guest_signup and assoc requests --- notebooks/api/0.8/00-load-data.ipynb | 2 +- .../0.8/07-domain-register-control-flow.ipynb | 6 +++--- .../data-owner/02-account-management.ipynb | 4 ++-- .../syft/service/settings/settings_service.py | 12 +++++------ packages/syft/tests/syft/api_test.py | 2 +- tests/integration/network/gateway_test.py | 20 +++++++++---------- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/notebooks/api/0.8/00-load-data.ipynb b/notebooks/api/0.8/00-load-data.ipynb index d1cedc7ca57..c61fab17f41 100644 --- a/notebooks/api/0.8/00-load-data.ipynb +++ b/notebooks/api/0.8/00-load-data.ipynb @@ -662,7 +662,7 @@ "### Create a new Data Scientist account on the Domain Server\n", "\n", "Signup is disabled by default.\n", - "An Admin/DO can enable it by `domain_client.settings.enable_guest_signup(enable=True)`\n", + "An Admin/DO can enable it by `domain_client.settings.allow_guest_signup(enable=True)`\n", "\n", "Refer to notebook [07-domain-register-control-flow](./07-domain-register-control-flow.ipynb) for more information." ] diff --git a/notebooks/api/0.8/07-domain-register-control-flow.ipynb b/notebooks/api/0.8/07-domain-register-control-flow.ipynb index 1c8721fa84d..5bd493a47c9 100644 --- a/notebooks/api/0.8/07-domain-register-control-flow.ipynb +++ b/notebooks/api/0.8/07-domain-register-control-flow.ipynb @@ -90,7 +90,7 @@ "# The assumed state of this test is a node with signup set to False\n", "# however if the tox task has set it to True you need to overwrite the setting\n", "# before running the tests\n", - "# root_client.settings.enable_guest_signup(enable=False)" + "# root_client.settings.allow_guest_signup(enable=False)" ] }, { @@ -184,7 +184,7 @@ "outputs": [], "source": [ "# Enable guest signups\n", - "root_client.settings.enable_guest_signup(enable=True)" + "root_client.settings.allow_guest_signup(enable=True)" ] }, { @@ -261,7 +261,7 @@ "outputs": [], "source": [ "# Refresh the root client\n", - "root_client.settings.enable_guest_signup(enable=False)" + "root_client.settings.allow_guest_signup(enable=False)" ] }, { diff --git a/notebooks/tutorials/data-owner/02-account-management.ipynb b/notebooks/tutorials/data-owner/02-account-management.ipynb index a38df5a3c35..a4e64b74698 100644 --- a/notebooks/tutorials/data-owner/02-account-management.ipynb +++ b/notebooks/tutorials/data-owner/02-account-management.ipynb @@ -379,7 +379,7 @@ "metadata": {}, "outputs": [], "source": [ - "client.settings.enable_guest_signup(enable=True)" + "client.settings.allow_guest_signup(enable=True)" ] }, { @@ -422,7 +422,7 @@ "metadata": {}, "outputs": [], "source": [ - "client.settings.enable_guest_signup(enable=False)" + "client.settings.allow_guest_signup(enable=False)" ] }, { diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index 918cc9826ec..4e54af69fb2 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -162,11 +162,11 @@ def disable_notifications( return notifier_service.turn_off(context=context) @service_method( - path="settings.enable_guest_signup", - name="enable_guest_signup", + path="settings.allow_guest_signup", + name="allow_guest_signup", warning=HighSideCRUDWarning(confirmation=True), ) - def enable_guest_signup( + def allow_guest_signup( self, context: AuthedServiceContext, enable: bool ) -> SyftSuccess | SyftError: """Enable/Disable Registration for Data Scientist or Guest Users.""" @@ -201,10 +201,10 @@ def enable_eager_execution( return SyftSuccess(message=f"Eager execution {message}") @service_method( - path="settings.enable_association_request_auto_approval", - name="enable_association_request_auto_approval", + path="settings.allow_association_request_auto_approval", + name="allow_association_request_auto_approval", ) - def enable_association_request_auto_approval( + def allow_association_request_auto_approval( self, context: AuthedServiceContext, enable: bool ) -> SyftSuccess | SyftError: new_settings = NodeSettingsUpdate(association_request_auto_approval=enable) diff --git a/packages/syft/tests/syft/api_test.py b/packages/syft/tests/syft/api_test.py index 3c1ddd25072..180060caffb 100644 --- a/packages/syft/tests/syft/api_test.py +++ b/packages/syft/tests/syft/api_test.py @@ -42,7 +42,7 @@ def my_func(x): def test_api_cache_invalidation_login(root_verify_key, worker): guest_client = worker.guest_client - worker.root_client.settings.enable_guest_signup(enable=True) + worker.root_client.settings.allow_guest_signup(enable=True) assert guest_client.register( name="q", email="a@b.org", password="aaa", password_verify="aaa" ) diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index 3c436cc707e..be72aae81e6 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -177,7 +177,7 @@ def test_dataset_search(set_env_var, gateway_port: int, domain_1_port: int) -> N port=domain_1_port, email="info@openmined.org", password="changethis" ) - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connect the domain to the gateway @@ -251,7 +251,7 @@ def test_domain_gateway_user_code( ) assert isinstance(user_create_res, SyftSuccess) - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # the domain client connects to the gateway @@ -304,7 +304,7 @@ def test_deleting_peers(set_env_var, domain_1_port: int, gateway_port: int) -> N ) # Enable automatic acceptance of association requests - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -372,7 +372,7 @@ def test_add_update_route_priority( _remove_existing_peers(gateway_client) # Enable automatic acceptance of association requests - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -456,7 +456,7 @@ def test_delete_route(set_env_var, gateway_port: int, domain_1_port: int) -> Non ) # Enable automatic acceptance of association requests - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -513,7 +513,7 @@ def test_add_update_route_priority_on_peer( _remove_existing_peers(gateway_client) # Enable automatic acceptance of association requests - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -586,7 +586,7 @@ def test_delete_route_on_peer( ) # Enable automatic acceptance of association requests - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -653,7 +653,7 @@ def test_update_route_priority( _remove_existing_peers(gateway_client) # Enable automatic acceptance of association requests - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -724,7 +724,7 @@ def test_update_route_priority_on_peer( _remove_existing_peers(gateway_client) # Enable automatic acceptance of association requests - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connecting the domain to the gateway @@ -794,7 +794,7 @@ def test_dataset_stream(set_env_var, gateway_port: int, domain_1_port: int) -> N _remove_existing_peers(domain_client) _remove_existing_peers(gateway_client) - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) # connect the domain to the gateway From df9446cb9e24ca6723b867639edaef1c8403a11f Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 21:23:07 +0200 Subject: [PATCH 16/21] fix: forgot one in gateway test --- tests/integration/local/gateway_local_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/local/gateway_local_test.py b/tests/integration/local/gateway_local_test.py index 5fd6f3f9f1f..a26e6ad35bb 100644 --- a/tests/integration/local/gateway_local_test.py +++ b/tests/integration/local/gateway_local_test.py @@ -141,7 +141,7 @@ def test_create_gateway( email="info@openmined.org", password="changethis", ) - res = gateway_client.settings.enable_association_request_auto_approval(enable=True) + res = gateway_client.settings.allow_association_request_auto_approval(enable=True) assert isinstance(res, SyftSuccess) domain_client: DomainClient = domain_webserver.login( From 49571958229dcbbc4d5f1cf4195ba1fedd6ae2ec Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 21:24:09 +0200 Subject: [PATCH 17/21] fix: and one more --- notebooks/tutorials/model-auditing/colab/01-user-log.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb b/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb index 5a414a80d33..036c21f9ed6 100644 --- a/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb +++ b/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb @@ -167,7 +167,7 @@ "metadata": {}, "outputs": [], "source": [ - "mo_client.settings.enable_guest_signup(enable=True)" + "mo_client.settings.allow_guest_signup(enable=True)" ] }, { From a91381d805f0fae76b1a4752cd6c4ca162ad3eb4 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 21:46:44 +0200 Subject: [PATCH 18/21] lint: ruff --- packages/syft/src/syft/service/action/action_object.py | 4 ++-- packages/syft/tests/syft/service/action/action_object_test.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index cea9ca9a905..8418b991392 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1505,7 +1505,7 @@ def _syft_add_pre_hooks__(self, eager_execution: bool): Add pre-hooks Args: - eager_execution: bool: If eager execution is enabled, hooks for + eager_execution: bool: If eager execution is enabled, hooks for tracing and executing the action on remote are added. """ @@ -1527,7 +1527,7 @@ def _syft_add_post_hooks__(self, eager_execution: bool): Add post-hooks Args: - eager_execution: bool: If eager execution is enabled, hooks for + eager_execution: bool: If eager execution is enabled, hooks for tracing and executing the action on remote are added. """ if eager_execution: diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index fe13a558e91..a595fdd0e8d 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -165,6 +165,7 @@ def test_actionobject_hooks_init(orig_obj: Any): assert make_action_side_effect in obj.syft_pre_hooks__[HOOK_ALWAYS] + def test_actionobject_add_pre_hooks(): # Eager execution is disabled by default obj = ActionObject.from_obj(1) @@ -181,6 +182,7 @@ def test_actionobject_add_pre_hooks(): assert send_action_side_effect in obj.syft_pre_hooks__[HOOK_ON_POINTERS] assert propagate_node_uid in obj.syft_post_hooks__[HOOK_ALWAYS] + @pytest.mark.parametrize( "orig_obj_op", [ From 5f556c690fa7e248a2bcf273ca51556bb72b2846 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 30 May 2024 21:54:14 +0200 Subject: [PATCH 19/21] fix: mypy complaints --- packages/syft/src/syft/service/action/action_object.py | 4 ++-- packages/syft/src/syft/service/action/plan.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 8418b991392..dffa3d3d9de 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1500,7 +1500,7 @@ def __post_init__(self) -> None: self.syft_history_hash = hash(self.id) - def _syft_add_pre_hooks__(self, eager_execution: bool): + def _syft_add_pre_hooks__(self, eager_execution: bool) -> None: """ Add pre-hooks @@ -1522,7 +1522,7 @@ def _syft_add_pre_hooks__(self, eager_execution: bool): if trace_action_side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: self.syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) - def _syft_add_post_hooks__(self, eager_execution: bool): + def _syft_add_post_hooks__(self, eager_execution: bool) -> None: """ Add post-hooks diff --git a/packages/syft/src/syft/service/action/plan.py b/packages/syft/src/syft/service/action/plan.py index 87730375473..2b89a436b64 100644 --- a/packages/syft/src/syft/service/action/plan.py +++ b/packages/syft/src/syft/service/action/plan.py @@ -66,9 +66,10 @@ def planify(func: Callable) -> ActionObject: ActionObject.add_trace_hook() worker = Worker.named(name="plan_building", reset=True, processes=0) client = worker.root_client - client.settings.enable_eager_execution(enable=True) if client is None: raise ValueError("Not able to get client for plan building") + if client.settings is not None: + client.settings.enable_eager_execution(enable=True) TraceResultRegistry.set_trace_result_for_current_thread(client=client) try: # TraceResult._client = client From 571ddebf79c39be758a16f0a434b424c0ba9a515 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 31 May 2024 10:12:18 +0200 Subject: [PATCH 20/21] fix: made metadata optional on get_api --- packages/syft/src/syft/client/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 85be22d311e..498c22e7536 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -244,7 +244,7 @@ def get_api( self, credentials: SyftSigningKey, communication_protocol: int, - metadata: NodeMetadataJSON | None, + metadata: NodeMetadataJSON | None = None, ) -> SyftAPI: params = { "verify_key": str(credentials.verify_key), @@ -385,7 +385,7 @@ def get_api( self, credentials: SyftSigningKey, communication_protocol: int, - metadata: NodeMetadataJSON | None, + metadata: NodeMetadataJSON | None = None, ) -> SyftAPI: # todo: its a bit odd to identify a user by its verify key maybe? if self.proxy_target_uid: From 3f2c1b3dfff6e9cfeacbae15a30b6ff6d40d981f Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 31 May 2024 11:15:32 +0200 Subject: [PATCH 21/21] fix: add ADMIN role check to eager execution endpoint --- packages/syft/src/syft/service/settings/settings_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index 6e392983b4b..35ef9262860 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -219,6 +219,7 @@ def allow_guest_signup( @service_method( path="settings.enable_eager_execution", name="enable_eager_execution", + roles=ADMIN_ROLE_LEVEL, warning=HighSideCRUDWarning(confirmation=True), ) def enable_eager_execution(