diff --git a/notebooks/Experimental/Rasswanth/gateway_test.ipynb b/notebooks/Experimental/Rasswanth/gateway_test.ipynb new file mode 100644 index 00000000000..91f6e817321 --- /dev/null +++ b/notebooks/Experimental/Rasswanth/gateway_test.ipynb @@ -0,0 +1,2458 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6b068788-a806-4da3-ae30-c3261628d446", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/rasswanth/PySyft\n" + ] + } + ], + "source": [ + "import syft as sy\n", + "import numpy as np\n", + "import uuid" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d66e3052-3484-4655-bf18-aac607af138f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as GUEST\n", + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# gateway_node = sy.orchestra.launch(name=\"gateway-1\", node_type=\"gateway\", dev_mode=True, reset=True)\n", + "# gt = gateway_node.client\n", + "gt = sy.login_as_guest(port=9081)\n", + "\n", + "# domain_node = sy.orchestra.launch(name=\"domain-1\", node_type=\"domain\", dev_mode=True, reset=True)\n", + "# do = domain_node.client.login(email=\"info@openmined.org\", password = \"changethis\")\n", + "do = sy.login(port=8081, email=\"info@openmined.org\", password = \"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7d2bb6a9-ee72-4643-bfd4-29cc5fae156e", + "metadata": {}, + "outputs": [], + "source": [ + "def random_hash() -> str:\n", + " return uuid.uuid4().hex[:16]\n", + "asset_name = random_hash()\n", + "asset = sy.Asset(\n", + " name=asset_name, data=np.array([1, 2, 3]), mock=sy.ActionObject.empty()\n", + ")\n", + "dataset_name = random_hash()\n", + "dataset = sy.Dataset(name=dataset_name, asset_list=[asset])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "91bed4af-09f3-4210-8551-a5b98f42d7bb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're approving a request on high side domain which may host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Would you like to proceed? [y/n]: y\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/1 [00:00SyftSuccess: Dataset uploaded to 'domain-1'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`
" + ], + "text/plain": [ + "SyftSuccess: Dataset uploaded to 'domain-1'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.upload_dataset(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "315b38ce-9dd3-423e-bdf3-cea136a6d602", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're registering a user to a high side domain, which could host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're registering a user to a high side domain, which could host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
SyftError: User already exists with email: sheldon@caltech.edu

" + ], + "text/plain": [ + "SyftError: User already exists with email: sheldon@caltech.edu" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.register(name=\"Sheldon Cooper\",\n", + " email=\"sheldon@caltech.edu\",\n", + " password=\"changethis\",\n", + " password_verify=\"changethis\",\n", + " institution=\"Caltech\",\n", + " website=\"https://www.caltech.edu/\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cc56ff55-e341-4cdc-bc45-1756883c0fa9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're performing an operation on high side domain, which could host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're performing an operation on high side domain, which could host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Would you like to proceed? \u001b[1;35m[y/n]\u001b[0m:  y\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
SyftSuccess: Connected domain to gateway-1 gateway

" + ], + "text/plain": [ + "SyftSuccess: Connected domain to gateway-1 gateway" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.connect_to_gateway(gt)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "8238fb40-880c-49b1-9835-5ef4a74ada89", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

NodePeer List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/markdown": [ + "```python\n", + "class ProxyClient:\n", + " id: str = 58d1d6afb8204208b68e6a34c3222ea7\n", + "\n", + "```" + ], + "text/plain": [ + "syft.client.gateway_client.ProxyClient" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gt.domains" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "ab6370bc-e222-4211-866c-5a0e87f4ed0c", + "metadata": {}, + "outputs": [], + "source": [ + "proxy_client = gt.domains[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c24de5c5-8c74-4618-9949-37af90c3f78c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "proxy_ds = proxy_client.login(email=\"sheldon@caltech.edu\", password=\"changethis\",\n", + " password_verify = \"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "22b1c74f-3cb5-440f-bddb-34ac7d14e039", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're performing an operation on high side domain, which could host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're performing an operation on high side domain, which could host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "asset = proxy_ds.datasets[0].assets[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "dc5d6a9a-b896-46a1-a105-63d8f77ad397", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: Syft function 'test' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" + ], + "text/plain": [ + "SyftSuccess: Syft function 'test' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "@sy.syft_function_single_use(asset=asset)\n", + "def test(asset):\n", + " import random\n", + " return random.randint(1,100)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "b2b5ffb3-00b4-4249-ac7b-b353cb461fc3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Warning: The result you see is computed on MOCK data.\n" + ] + }, + { + "data": { + "text/plain": [ + "63" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test(asset=asset)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "e07a6153-2cd0-43c4-9ced-2951799689db", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "

Request

\n", + "

Id: 81f811239ba84c19b9f2f636012a7ef3

\n", + "

Request time: 2023-11-28 02:43:02

\n", + " \n", + " \n", + "

Changes: Request to change test to permission RequestStatus.APPROVED.

\n", + "

Status: RequestStatus.PENDING

\n", + "

Requested on: Domain-1 of type Domain

\n", + "

Requested by: Jane Doe (sheldon@caltech.edu) Institution: Caltech

\n", + "
\n", + "\n", + " " + ], + "text/markdown": [ + "```python\n", + "class Request:\n", + " id: str = 81f811239ba84c19b9f2f636012a7ef3\n", + " request_time: str = 2023-11-28 02:43:02\n", + " updated_at: str = None\n", + " status: str = RequestStatus.PENDING\n", + " changes: str = ['Request to change test to permission RequestStatus.APPROVED']\n", + " requesting_user_verify_key: str = e0edcb2e780b4dee4a8533082b2f895510b2ee3a7324a3fb9996bd6cef41b11c\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.request.request.Request" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "proxy_ds.code.request_code_execution(test)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "e1f5459c-e1c4-4903-abbd-4df365e99f4d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Request List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.requests" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "2c772bfa-206a-4c6d-a242-8f7c762c807c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're approving a request on high side domain which may host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Would you like to proceed? [y/n]: y\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Request approved for domain domain-1\n" + ] + }, + { + "data": { + "text/html": [ + "
SyftSuccess: Request 81f811239ba84c19b9f2f636012a7ef3 changes applied

" + ], + "text/plain": [ + "SyftSuccess: Request 81f811239ba84c19b9f2f636012a7ef3 changes applied" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.requests[1].approve()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "533ae51c-2489-4fc6-b8f1-9073be8be421", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Request List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.requests" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "0e4a4299-65e8-4924-9a45-bb664a6e4d0c", + "metadata": {}, + "outputs": [], + "source": [ + "result = proxy_ds.code.test(asset=asset)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "b5dccfd5-055c-42ec-8436-8c9d0ef48c93", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftError: Failed to retrieve with Error: 502 Server Error: Bad Gateway for url: http://localhost:9081/blob/a7202ccf-d0d2-4346-a652-acfb89c86462/f3eb4b34e8804282b58c4f31ecb34ff6?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20231128%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20231128T024351Z&X-Amz-Expires=1800&X-Amz-SignedHeaders=host&X-Amz-Signature=a265c98e538a040c3fca40ad899a8d63250c42c6558be5fe9d9d7d00d8dc6d50

" + ], + "text/plain": [ + "SyftError: Failed to retrieve with Error: 502 Server Error: Bad Gateway for url: http://localhost:9081/blob/a7202ccf-d0d2-4346-a652-acfb89c86462/f3eb4b34e8804282b58c4f31ecb34ff6?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20231128%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20231128T024351Z&X-Amz-Expires=1800&X-Amz-SignedHeaders=host&X-Amz-Signature=a265c98e538a040c3fca40ad899a8d63250c42c6558be5fe9d9d7d00d8dc6d50" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.get()" + ] + }, + { + "cell_type": "markdown", + "id": "04274391-3d70-4c10-9f9f-c59f886e9cb2", + "metadata": {}, + "source": [ + "------------------------------------------------------------------------------------------------------------" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ae9b5e6-b71a-4881-a282-c55f430b3a99", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41b42566-3d87-4bb4-aa8c-775ef816857c", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b11b5de-3c6a-4ed4-8156-58d0fbd9a6ae", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7af25df0-3ee9-4c30-b2f3-789a0679f037", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7bf3611-3a44-4663-beeb-073400957ade", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64e41583-6eae-493b-a85a-2a7923246f6e", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 30597a89d03..0e7f5e82aa6 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -843,6 +843,19 @@ def forward_message( signed_result = client.connection.make_call(api_call) result = debox_signed_syftapicall_response(signed_result=signed_result) + # relative + from ..store.blob_storage import BlobRetrievalByURL + + # In the case of blob storage, the gateway downloads the result and then passes it to + # the proxy client + if isinstance(result, BlobRetrievalByURL): + blob_route = client.api.connection.to_blob_route( + result.url.url_path + ) + result.url = blob_route + final_res = result.read() + return final_res + return result return SyftError(message=(f"Node has no route to {node_uid}")) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index b833fcbf09b..94ef6f66d12 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -32,6 +32,7 @@ from ...serde.serializable import serializable from ...serde.serialize import _serialize as serialize from ...service.response import SyftError +from ...store.blob_storage import BlobRetrieval from ...store.linked_obj import LinkedObject from ...types.blob_storage import CreateBlobStorageEntry from ...types.datetime import DateTime @@ -511,8 +512,22 @@ def reload_cache(self): blob_retrieval_object, ) return blob_retrieval_object - self.syft_action_data_cache = blob_retrieval_object.read() - self.syft_action_data_type = type(self.syft_action_data) + + if isinstance(blob_retrieval_object, SyftError): + raise SyftException( + message=f"Failed to retrieve object from blob storage: {blob_retrieval_object.message}" + ) + + elif isinstance(blob_retrieval_object, BlobRetrieval): + # TODO: This change is temporary to for gateway to be compatible with the new blob storage + self.syft_action_data_cache = blob_retrieval_object.read() + self.syft_action_data_type = type(self.syft_action_data) + else: + # In the case of gateway, we directly receive the actual object + # TODO: The ideal solution would be to stream the object from the domain through the gateway + # Currently , we are just passing the object as it is, which would be fixed later. + self.syft_action_data_cache = blob_retrieval_object + self.syft_action_data_type = type(self.syft_action_data) def _save_to_blob_storage_(self, data: Any) -> None: if not isinstance(data, ActionDataEmpty): diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index e5c1e4e6adf..81eb28a99c3 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -1,9 +1,17 @@ +# stdlib +from textwrap import dedent +import uuid + +# third party +import numpy as np + # syft absolute import syft as sy from syft.abstract_node import NodeType from syft.client.domain_client import DomainClient from syft.client.gateway_client import GatewayClient from syft.service.network.node_peer import NodePeer +from syft.service.request.request import Request from syft.service.response import SyftSuccess from syft.service.user.user_roles import ServiceRole @@ -52,3 +60,69 @@ def test_domain_connect_to_gateway(domain_1_port, gateway_port): assert ( proxy_domain_client.api.endpoints.keys() == domain_client.api.endpoints.keys() ) + + +def random_hash() -> str: + return uuid.uuid4().hex[:16] + + +def test_domain_gateway_user_code(domain_1_port, gateway_port): + gateway_client: GatewayClient = sy.login_as_guest(port=gateway_port) + + domain_client: DomainClient = sy.login( + port=domain_1_port, email="info@openmined.org", password="changethis" + ) + + input_data = np.array([1, 2, 3]) + mock_data = np.array([4, 5, 6]) + + asset_name = random_hash() + asset = sy.Asset(name=asset_name, data=input_data, mock=mock_data) + dataset_name = random_hash() + dataset = sy.Dataset(name=dataset_name, asset_list=[asset]) + + dataset_res = domain_client.upload_dataset(dataset) + + assert isinstance(dataset_res, SyftSuccess) + + user_create_res = domain_client.register( + name="Sheldon Cooper", + email="sheldon@caltech.edu", + password="changethis", + password_verify="changethis", + institution="Caltech", + website="https://www.caltech.edu/", + ) + + assert isinstance(user_create_res, SyftSuccess) + + gateway_con_res = domain_client.connect_to_gateway(gateway_client) + assert isinstance(gateway_con_res, SyftSuccess) + + proxy_client = gateway_client.domains[0] + + proxy_ds = proxy_client.login( + email="sheldon@caltech.edu", password="changethis", password_verify="changethis" + ) + + asset = proxy_ds.datasets[0].assets[0] + + @sy.syft_function_single_use(asset=asset) + def test_function(asset): + return asset + 1 + + test_function.code = dedent(test_function.code) + + request_res = proxy_ds.code.request_code_execution(test_function) + assert isinstance(request_res, Request) + + assert len(domain_client.requests.get_all()) == 1 + + req_approve_res = domain_client.requests[-1].approve() + assert isinstance(req_approve_res, SyftSuccess) + + result = proxy_ds.code.test_function(asset=asset) + + final_result = result.get() + + assert (final_result == input_data + 1).all() diff --git a/tox.ini b/tox.ini index 24c639f2dc3..726e9fdcdd9 100644 --- a/tox.ini +++ b/tox.ini @@ -793,7 +793,9 @@ commands = pytest --nbmake notebooks/api/0.8 -p no:randomly -vvvv #Integration + Gateway Connection Tests - pytest tests/integration/network -p no:randomly -vvvv + # Gateway tests are not run in kuberetes, as currently,it does not have a way to configure + # high/low side warning flag. + pytest tests/integration/network -k "not test_domain_gateway_user_code" -p no:randomly -vvvv # deleting clusters created bash -c "k3d cluster delete test-gateway-1 || true"