From 0c98c47ff6ebccd8368a0deb7545452fac2950a0 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 28 Nov 2023 07:52:09 +0530 Subject: [PATCH 1/8] add notebook for testing gateway bug --- .../Experimental/Rasswanth/gateway_test.ipynb | 2458 +++++++++++++++++ 1 file changed, 2458 insertions(+) create mode 100644 notebooks/Experimental/Rasswanth/gateway_test.ipynb diff --git a/notebooks/Experimental/Rasswanth/gateway_test.ipynb b/notebooks/Experimental/Rasswanth/gateway_test.ipynb new file mode 100644 index 00000000000..08da0307423 --- /dev/null +++ b/notebooks/Experimental/Rasswanth/gateway_test.ipynb @@ -0,0 +1,2458 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6b068788-a806-4da3-ae30-c3261628d446", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/rasswanth/PySyft\n" + ] + } + ], + "source": [ + "import syft as sy" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d66e3052-3484-4655-bf18-aac607af138f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as GUEST\n", + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "gt = sy.login_as_guest(port=9081)\n", + "do = sy.login(port=8081, email=\"info@openmined.org\", password = \"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "315b38ce-9dd3-423e-bdf3-cea136a6d602", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're registering a user to a high side domain, which could host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're registering a user to a high side domain, which could host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
SyftSuccess: User 'Jane Doe' successfully registered! To see users, run `[your_client].users`

" + ], + "text/plain": [ + "SyftSuccess: User 'Jane Doe' successfully registered! To see users, run `[your_client].users`" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.register(name=\"Sheldon Cooper\",\n", + " email=\"sheldon@caltech.edu\",\n", + " password=\"changethis\",\n", + " password_verify=\"changethis\",\n", + " institution=\"Caltech\",\n", + " website=\"https://www.caltech.edu/\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cc56ff55-e341-4cdc-bc45-1756883c0fa9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're performing an operation on high side domain, which could host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're performing an operation on high side domain, which could host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Would you like to proceed? \u001b[1;35m[y/n]\u001b[0m:  y\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
SyftSuccess: Connected domain to gateway-1 gateway

" + ], + "text/plain": [ + "SyftSuccess: Connected domain to gateway-1 gateway" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.connect_to_gateway(gt)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8238fb40-880c-49b1-9835-5ef4a74ada89", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

NodePeer List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/markdown": [ + "```python\n", + "class ProxyClient:\n", + " id: str = 59f979ae908446afb1a3a4e539572363\n", + "\n", + "```" + ], + "text/plain": [ + "syft.client.gateway_client.ProxyClient" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gt.domains" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ab6370bc-e222-4211-866c-5a0e87f4ed0c", + "metadata": {}, + "outputs": [], + "source": [ + "proxy_client = gt.domains[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c24de5c5-8c74-4618-9949-37af90c3f78c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "proxy_ds = proxy_client.login(email=\"sheldon@caltech.edu\", password=\"changethis\",\n", + " password_verify = \"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "dc5d6a9a-b896-46a1-a105-63d8f77ad397", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: Syft function 'test' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" + ], + "text/plain": [ + "SyftSuccess: Syft function 'test' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "@sy.syft_function_single_use()\n", + "def test():\n", + " import random\n", + " return random.randint(1,100)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "b2b5ffb3-00b4-4249-ac7b-b353cb461fc3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "60" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e07a6153-2cd0-43c4-9ced-2951799689db", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "

Request

\n", + "

Id: abc600e3980b49db85a0a1b012ddd543

\n", + "

Request time: 2023-11-27 08:13:35

\n", + " \n", + " \n", + "

Changes: Request to change test to permission RequestStatus.APPROVED.

\n", + "

Status: RequestStatus.PENDING

\n", + "

Requested on: Domain-1 of type Domain

\n", + "

Requested by: Jane Doe (sheldon@caltech.edu) Institution: Caltech

\n", + "
\n", + "\n", + " " + ], + "text/markdown": [ + "```python\n", + "class Request:\n", + " id: str = abc600e3980b49db85a0a1b012ddd543\n", + " request_time: str = 2023-11-27 08:13:35\n", + " updated_at: str = None\n", + " status: str = RequestStatus.PENDING\n", + " changes: str = ['Request to change test to permission RequestStatus.APPROVED']\n", + " requesting_user_verify_key: str = e0edcb2e780b4dee4a8533082b2f895510b2ee3a7324a3fb9996bd6cef41b11c\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.request.request.Request" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "proxy_ds.code.request_code_execution(test)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "e1f5459c-e1c4-4903-abbd-4df365e99f4d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Request List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.requests" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "2c772bfa-206a-4c6d-a242-8f7c762c807c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're approving a request on high side domain which may host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Would you like to proceed? [y/n]: y\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Request approved for domain domain-1\n" + ] + }, + { + "data": { + "text/html": [ + "
SyftSuccess: Request abc600e3980b49db85a0a1b012ddd543 changes applied

" + ], + "text/plain": [ + "SyftSuccess: Request abc600e3980b49db85a0a1b012ddd543 changes applied" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.requests[-1].approve()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "533ae51c-2489-4fc6-b8f1-9073be8be421", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Request List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.requests" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "0e4a4299-65e8-4924-9a45-bb664a6e4d0c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftError: Failed to run. NodeIdentity

" + ], + "text/plain": [ + "SyftError: Failed to run. NodeIdentity " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "proxy_ds.code.test()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "c0683641-f1e1-438b-a363-133e34459daa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "

Request

\n", + "

Id: 6632bcf8e53b4df8b66411e14b249974

\n", + "

Request time: 2023-11-27 08:15:41

\n", + " \n", + " \n", + "

Changes: Request to change test to permission RequestStatus.APPROVED.

\n", + "

Status: RequestStatus.PENDING

\n", + "

Requested on: Domain-1 of type Domain

\n", + "

Requested by: Jane Doe (info@openmined.org)

\n", + "
\n", + "\n", + " " + ], + "text/markdown": [ + "```python\n", + "class Request:\n", + " id: str = 6632bcf8e53b4df8b66411e14b249974\n", + " request_time: str = 2023-11-27 08:15:41\n", + " updated_at: str = None\n", + " status: str = RequestStatus.PENDING\n", + " changes: str = ['Request to change test to permission RequestStatus.APPROVED']\n", + " requesting_user_verify_key: str = 4acda4f206469e0ffab0ce8663910d245b07d279127cb17f2396671683823f4c\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.request.request.Request" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.code.request_code_execution(test)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "646653dd-14b6-4216-9c64-510458c190b3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're approving a request on high side domain which may host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Would you like to proceed? [y/n]: y\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Request approved for domain domain-1\n" + ] + }, + { + "data": { + "text/html": [ + "
SyftSuccess: Request abc600e3980b49db85a0a1b012ddd543 changes applied

" + ], + "text/plain": [ + "SyftSuccess: Request abc600e3980b49db85a0a1b012ddd543 changes applied" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.requests[-1].approve()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "50561e4c-643b-47dc-a490-e1c51798f781", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're approving a request on high side domain which may host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Would you like to proceed? [y/n]: y\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Request approved for domain domain-1\n" + ] + }, + { + "data": { + "text/html": [ + "
SyftSuccess: Request 6632bcf8e53b4df8b66411e14b249974 changes applied

" + ], + "text/plain": [ + "SyftSuccess: Request 6632bcf8e53b4df8b66411e14b249974 changes applied" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.requests[0].approve()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "456dca4d-0884-4c02-8175-6b923838ae47", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftError: Failed to run. NodeIdentity

" + ], + "text/plain": [ + "SyftError: Failed to run. NodeIdentity " + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "do.code.test()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64e41583-6eae-493b-a85a-2a7923246f6e", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 1193dfc1577fc0e828472e22160ac1084a936713 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:09:30 +0530 Subject: [PATCH 2/8] updated notebook with test dataset --- .../Experimental/Rasswanth/gateway_test.ipynb | 2357 +---------------- 1 file changed, 83 insertions(+), 2274 deletions(-) diff --git a/notebooks/Experimental/Rasswanth/gateway_test.ipynb b/notebooks/Experimental/Rasswanth/gateway_test.ipynb index 08da0307423..cbb9941baaa 100644 --- a/notebooks/Experimental/Rasswanth/gateway_test.ipynb +++ b/notebooks/Experimental/Rasswanth/gateway_test.ipynb @@ -2,86 +2,65 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "6b068788-a806-4da3-ae30-c3261628d446", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/rasswanth/PySyft\n" - ] - } - ], + "outputs": [], "source": [ - "import syft as sy" + "import syft as sy\n", + "import numpy as np\n", + "import uuid" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "d66e3052-3484-4655-bf18-aac607af138f", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as GUEST\n", - "Logged into as \n" - ] - }, - { - "data": { - "text/html": [ - "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" - ], - "text/plain": [ - "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ + "# gateway_node = sy.orchestra.launch(name=\"gateway-1\", node_type=\"gateway\", dev_mode=True, reset=True)\n", + "# gt = gateway_node.client\n", "gt = sy.login_as_guest(port=9081)\n", + "\n", + "# domain_node = sy.orchestra.launch(name=\"domain-1\", node_type=\"domain\", dev_mode=True, reset=True)\n", + "# do = domain_node.client.login(email=\"info@openmined.org\", password = \"changethis\")\n", "do = sy.login(port=8081, email=\"info@openmined.org\", password = \"changethis\")" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, + "id": "7d2bb6a9-ee72-4643-bfd4-29cc5fae156e", + "metadata": {}, + "outputs": [], + "source": [ + "def random_hash() -> str:\n", + " return uuid.uuid4().hex[:16]\n", + "asset_name = random_hash()\n", + "asset = sy.Asset(\n", + " name=asset_name, data=np.array([1, 2, 3]), mock=sy.ActionObject.empty()\n", + ")\n", + "dataset_name = random_hash()\n", + "dataset = sy.Dataset(name=dataset_name, asset_list=[asset])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91bed4af-09f3-4210-8551-a5b98f42d7bb", + "metadata": {}, + "outputs": [], + "source": [ + "do.upload_dataset(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "315b38ce-9dd3-423e-bdf3-cea136a6d602", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftWarning: You're registering a user to a high side domain, which could host datasets with private information.

" - ], - "text/plain": [ - "SyftWarning: You're registering a user to a high side domain, which could host datasets with private information." - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
SyftSuccess: User 'Jane Doe' successfully registered! To see users, run `[your_client].users`

" - ], - "text/plain": [ - "SyftSuccess: User 'Jane Doe' successfully registered! To see users, run `[your_client].users`" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.register(name=\"Sheldon Cooper\",\n", " email=\"sheldon@caltech.edu\",\n", @@ -93,695 +72,27 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "cc56ff55-e341-4cdc-bc45-1756883c0fa9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftWarning: You're performing an operation on high side domain, which could host datasets with private information.

" - ], - "text/plain": [ - "SyftWarning: You're performing an operation on high side domain, which could host datasets with private information." - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n"
-      ],
-      "text/plain": []
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdin",
-     "output_type": "stream",
-     "text": [
-      "Would you like to proceed? \u001b[1;35m[y/n]\u001b[0m:  y\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "
SyftSuccess: Connected domain to gateway-1 gateway

" - ], - "text/plain": [ - "SyftSuccess: Connected domain to gateway-1 gateway" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.connect_to_gateway(gt)" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "8238fb40-880c-49b1-9835-5ef4a74ada89", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "

NodePeer List

\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - "\n", - "

0

\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - " \n" - ], - "text/markdown": [ - "```python\n", - "class ProxyClient:\n", - " id: str = 59f979ae908446afb1a3a4e539572363\n", - "\n", - "```" - ], - "text/plain": [ - "syft.client.gateway_client.ProxyClient" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "gt.domains" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "ab6370bc-e222-4211-866c-5a0e87f4ed0c", "metadata": {}, "outputs": [], @@ -791,30 +102,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "c24de5c5-8c74-4618-9949-37af90c3f78c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - }, - { - "data": { - "text/html": [ - "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" - ], - "text/plain": [ - "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "proxy_ds = proxy_client.login(email=\"sheldon@caltech.edu\", password=\"changethis\",\n", " password_verify = \"changethis\")" @@ -822,1605 +113,123 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, + "id": "22b1c74f-3cb5-440f-bddb-34ac7d14e039", + "metadata": {}, + "outputs": [], + "source": [ + "asset = proxy_ds.datasets[0].assets[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "dc5d6a9a-b896-46a1-a105-63d8f77ad397", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftSuccess: Syft function 'test' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" - ], - "text/plain": [ - "SyftSuccess: Syft function 'test' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "@sy.syft_function_single_use()\n", - "def test():\n", + "@sy.syft_function_single_use(asset=asset)\n", + "def test(asset):\n", " import random\n", " return random.randint(1,100)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "b2b5ffb3-00b4-4249-ac7b-b353cb461fc3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "60" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "test()" + "test(asset=asset)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "e07a6153-2cd0-43c4-9ced-2951799689db", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "
\n", - "

Request

\n", - "

Id: abc600e3980b49db85a0a1b012ddd543

\n", - "

Request time: 2023-11-27 08:13:35

\n", - " \n", - " \n", - "

Changes: Request to change test to permission RequestStatus.APPROVED.

\n", - "

Status: RequestStatus.PENDING

\n", - "

Requested on: Domain-1 of type Domain

\n", - "

Requested by: Jane Doe (sheldon@caltech.edu) Institution: Caltech

\n", - "
\n", - "\n", - " " - ], - "text/markdown": [ - "```python\n", - "class Request:\n", - " id: str = abc600e3980b49db85a0a1b012ddd543\n", - " request_time: str = 2023-11-27 08:13:35\n", - " updated_at: str = None\n", - " status: str = RequestStatus.PENDING\n", - " changes: str = ['Request to change test to permission RequestStatus.APPROVED']\n", - " requesting_user_verify_key: str = e0edcb2e780b4dee4a8533082b2f895510b2ee3a7324a3fb9996bd6cef41b11c\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.request.request.Request" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "proxy_ds.code.request_code_execution(test)" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "e1f5459c-e1c4-4903-abbd-4df365e99f4d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "

Request List

\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - "\n", - "

0

\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - " \n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.requests" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "2c772bfa-206a-4c6d-a242-8f7c762c807c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" - ], - "text/plain": [ - "SyftWarning: You're approving a request on high side domain which may host datasets with private information." - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdin", - "output_type": "stream", - "text": [ - "Would you like to proceed? [y/n]: y\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Request approved for domain domain-1\n" - ] - }, - { - "data": { - "text/html": [ - "
SyftSuccess: Request abc600e3980b49db85a0a1b012ddd543 changes applied

" - ], - "text/plain": [ - "SyftSuccess: Request abc600e3980b49db85a0a1b012ddd543 changes applied" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.requests[-1].approve()" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "533ae51c-2489-4fc6-b8f1-9073be8be421", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "

Request List

\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - "\n", - "

0

\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - " \n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.requests" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "0e4a4299-65e8-4924-9a45-bb664a6e4d0c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftError: Failed to run. NodeIdentity

" - ], - "text/plain": [ - "SyftError: Failed to run. NodeIdentity " - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "proxy_ds.code.test()" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "c0683641-f1e1-438b-a363-133e34459daa", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "
\n", - "

Request

\n", - "

Id: 6632bcf8e53b4df8b66411e14b249974

\n", - "

Request time: 2023-11-27 08:15:41

\n", - " \n", - " \n", - "

Changes: Request to change test to permission RequestStatus.APPROVED.

\n", - "

Status: RequestStatus.PENDING

\n", - "

Requested on: Domain-1 of type Domain

\n", - "

Requested by: Jane Doe (info@openmined.org)

\n", - "
\n", - "\n", - " " - ], - "text/markdown": [ - "```python\n", - "class Request:\n", - " id: str = 6632bcf8e53b4df8b66411e14b249974\n", - " request_time: str = 2023-11-27 08:15:41\n", - " updated_at: str = None\n", - " status: str = RequestStatus.PENDING\n", - " changes: str = ['Request to change test to permission RequestStatus.APPROVED']\n", - " requesting_user_verify_key: str = 4acda4f206469e0ffab0ce8663910d245b07d279127cb17f2396671683823f4c\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.request.request.Request" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.code.request_code_execution(test)" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "646653dd-14b6-4216-9c64-510458c190b3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" - ], - "text/plain": [ - "SyftWarning: You're approving a request on high side domain which may host datasets with private information." - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdin", - "output_type": "stream", - "text": [ - "Would you like to proceed? [y/n]: y\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Request approved for domain domain-1\n" - ] - }, - { - "data": { - "text/html": [ - "
SyftSuccess: Request abc600e3980b49db85a0a1b012ddd543 changes applied

" - ], - "text/plain": [ - "SyftSuccess: Request abc600e3980b49db85a0a1b012ddd543 changes applied" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.requests[-1].approve()" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "50561e4c-643b-47dc-a490-e1c51798f781", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" - ], - "text/plain": [ - "SyftWarning: You're approving a request on high side domain which may host datasets with private information." - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdin", - "output_type": "stream", - "text": [ - "Would you like to proceed? [y/n]: y\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Request approved for domain domain-1\n" - ] - }, - { - "data": { - "text/html": [ - "
SyftSuccess: Request 6632bcf8e53b4df8b66411e14b249974 changes applied

" - ], - "text/plain": [ - "SyftSuccess: Request 6632bcf8e53b4df8b66411e14b249974 changes applied" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.requests[0].approve()" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "456dca4d-0884-4c02-8175-6b923838ae47", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftError: Failed to run. NodeIdentity

" - ], - "text/plain": [ - "SyftError: Failed to run. NodeIdentity " - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "do.code.test()" ] From 03de9e6ac098620a9365a0dfb1fdbc552ace7409 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:14:41 +0530 Subject: [PATCH 3/8] final reproducible notebook --- .../Experimental/Rasswanth/gateway_test.ipynb | 2289 ++++++++++++++++- 1 file changed, 2240 insertions(+), 49 deletions(-) diff --git a/notebooks/Experimental/Rasswanth/gateway_test.ipynb b/notebooks/Experimental/Rasswanth/gateway_test.ipynb index cbb9941baaa..91f6e817321 100644 --- a/notebooks/Experimental/Rasswanth/gateway_test.ipynb +++ b/notebooks/Experimental/Rasswanth/gateway_test.ipynb @@ -2,10 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "6b068788-a806-4da3-ae30-c3261628d446", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/rasswanth/PySyft\n" + ] + } + ], "source": [ "import syft as sy\n", "import numpy as np\n", @@ -14,10 +22,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "d66e3052-3484-4655-bf18-aac607af138f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as GUEST\n", + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# gateway_node = sy.orchestra.launch(name=\"gateway-1\", node_type=\"gateway\", dev_mode=True, reset=True)\n", "# gt = gateway_node.client\n", @@ -30,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "7d2bb6a9-ee72-4643-bfd4-29cc5fae156e", "metadata": {}, "outputs": [], @@ -47,20 +76,100 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "91bed4af-09f3-4210-8551-a5b98f42d7bb", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're approving a request on high side domain which may host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Would you like to proceed? [y/n]: y\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/1 [00:00SyftSuccess: Dataset uploaded to 'domain-1'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`
" + ], + "text/plain": [ + "SyftSuccess: Dataset uploaded to 'domain-1'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "do.upload_dataset(dataset)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "315b38ce-9dd3-423e-bdf3-cea136a6d602", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're registering a user to a high side domain, which could host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're registering a user to a high side domain, which could host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
SyftError: User already exists with email: sheldon@caltech.edu

" + ], + "text/plain": [ + "SyftError: User already exists with email: sheldon@caltech.edu" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "do.register(name=\"Sheldon Cooper\",\n", " email=\"sheldon@caltech.edu\",\n", @@ -72,27 +181,695 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "cc56ff55-e341-4cdc-bc45-1756883c0fa9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're performing an operation on high side domain, which could host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're performing an operation on high side domain, which could host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Would you like to proceed? \u001b[1;35m[y/n]\u001b[0m:  y\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
SyftSuccess: Connected domain to gateway-1 gateway

" + ], + "text/plain": [ + "SyftSuccess: Connected domain to gateway-1 gateway" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "do.connect_to_gateway(gt)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "8238fb40-880c-49b1-9835-5ef4a74ada89", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

NodePeer List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/markdown": [ + "```python\n", + "class ProxyClient:\n", + " id: str = 58d1d6afb8204208b68e6a34c3222ea7\n", + "\n", + "```" + ], + "text/plain": [ + "syft.client.gateway_client.ProxyClient" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "gt.domains" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "ab6370bc-e222-4211-866c-5a0e87f4ed0c", "metadata": {}, "outputs": [], @@ -102,10 +879,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "c24de5c5-8c74-4618-9949-37af90c3f78c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "proxy_ds = proxy_client.login(email=\"sheldon@caltech.edu\", password=\"changethis\",\n", " password_verify = \"changethis\")" @@ -113,20 +910,46 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "22b1c74f-3cb5-440f-bddb-34ac7d14e039", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're performing an operation on high side domain, which could host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're performing an operation on high side domain, which could host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "asset = proxy_ds.datasets[0].assets[0]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "dc5d6a9a-b896-46a1-a105-63d8f77ad397", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: Syft function 'test' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" + ], + "text/plain": [ + "SyftSuccess: Syft function 'test' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "@sy.syft_function_single_use(asset=asset)\n", "def test(asset):\n", @@ -136,103 +959,1471 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "b2b5ffb3-00b4-4249-ac7b-b353cb461fc3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Warning: The result you see is computed on MOCK data.\n" + ] + }, + { + "data": { + "text/plain": [ + "63" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "test(asset=asset)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "e07a6153-2cd0-43c4-9ced-2951799689db", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "

Request

\n", + "

Id: 81f811239ba84c19b9f2f636012a7ef3

\n", + "

Request time: 2023-11-28 02:43:02

\n", + " \n", + " \n", + "

Changes: Request to change test to permission RequestStatus.APPROVED.

\n", + "

Status: RequestStatus.PENDING

\n", + "

Requested on: Domain-1 of type Domain

\n", + "

Requested by: Jane Doe (sheldon@caltech.edu) Institution: Caltech

\n", + "
\n", + "\n", + " " + ], + "text/markdown": [ + "```python\n", + "class Request:\n", + " id: str = 81f811239ba84c19b9f2f636012a7ef3\n", + " request_time: str = 2023-11-28 02:43:02\n", + " updated_at: str = None\n", + " status: str = RequestStatus.PENDING\n", + " changes: str = ['Request to change test to permission RequestStatus.APPROVED']\n", + " requesting_user_verify_key: str = e0edcb2e780b4dee4a8533082b2f895510b2ee3a7324a3fb9996bd6cef41b11c\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.request.request.Request" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "proxy_ds.code.request_code_execution(test)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "e1f5459c-e1c4-4903-abbd-4df365e99f4d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Request List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "do.requests" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "2c772bfa-206a-4c6d-a242-8f7c762c807c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
SyftWarning: You're approving a request on high side domain which may host datasets with private information.

" + ], + "text/plain": [ + "SyftWarning: You're approving a request on high side domain which may host datasets with private information." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Would you like to proceed? [y/n]: y\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Request approved for domain domain-1\n" + ] + }, + { + "data": { + "text/html": [ + "
SyftSuccess: Request 81f811239ba84c19b9f2f636012a7ef3 changes applied

" + ], + "text/plain": [ + "SyftSuccess: Request 81f811239ba84c19b9f2f636012a7ef3 changes applied" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "do.requests[-1].approve()" + "do.requests[1].approve()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "533ae51c-2489-4fc6-b8f1-9073be8be421", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Request List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "do.requests" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "0e4a4299-65e8-4924-9a45-bb664a6e4d0c", "metadata": {}, "outputs": [], "source": [ - "proxy_ds.code.test()" + "result = proxy_ds.code.test(asset=asset)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "c0683641-f1e1-438b-a363-133e34459daa", + "execution_count": 27, + "id": "b5dccfd5-055c-42ec-8436-8c9d0ef48c93", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftError: Failed to retrieve with Error: 502 Server Error: Bad Gateway for url: http://localhost:9081/blob/a7202ccf-d0d2-4346-a652-acfb89c86462/f3eb4b34e8804282b58c4f31ecb34ff6?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20231128%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20231128T024351Z&X-Amz-Expires=1800&X-Amz-SignedHeaders=host&X-Amz-Signature=a265c98e538a040c3fca40ad899a8d63250c42c6558be5fe9d9d7d00d8dc6d50

" + ], + "text/plain": [ + "SyftError: Failed to retrieve with Error: 502 Server Error: Bad Gateway for url: http://localhost:9081/blob/a7202ccf-d0d2-4346-a652-acfb89c86462/f3eb4b34e8804282b58c4f31ecb34ff6?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20231128%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20231128T024351Z&X-Amz-Expires=1800&X-Amz-SignedHeaders=host&X-Amz-Signature=a265c98e538a040c3fca40ad899a8d63250c42c6558be5fe9d9d7d00d8dc6d50" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.get()" + ] + }, + { + "cell_type": "markdown", + "id": "04274391-3d70-4c10-9f9f-c59f886e9cb2", "metadata": {}, - "outputs": [], "source": [ - "do.code.request_code_execution(test)" + "------------------------------------------------------------------------------------------------------------" ] }, { "cell_type": "code", "execution_count": null, - "id": "646653dd-14b6-4216-9c64-510458c190b3", + "id": "8ae9b5e6-b71a-4881-a282-c55f430b3a99", "metadata": {}, "outputs": [], - "source": [ - "do.requests[-1].approve()" - ] + "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "50561e4c-643b-47dc-a490-e1c51798f781", + "id": "41b42566-3d87-4bb4-aa8c-775ef816857c", "metadata": {}, "outputs": [], - "source": [ - "do.requests[0].approve()" - ] + "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "456dca4d-0884-4c02-8175-6b923838ae47", + "id": "7b11b5de-3c6a-4ed4-8156-58d0fbd9a6ae", "metadata": {}, "outputs": [], - "source": [ - "do.code.test()" - ] + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7af25df0-3ee9-4c30-b2f3-789a0679f037", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7bf3611-3a44-4663-beeb-073400957ade", + "metadata": {}, + "outputs": [], + "source": [] }, { "cell_type": "code", From f2d5060fe940dd76cc05425463485126244e566d Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 29 Nov 2023 12:05:41 +0530 Subject: [PATCH 4/8] add a temporary fix to retrive blob storage at the gateway level --- packages/syft/src/syft/node/node.py | 13 +++++++++++++ .../syft/src/syft/service/action/action_object.py | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 30597a89d03..0e7f5e82aa6 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -843,6 +843,19 @@ def forward_message( signed_result = client.connection.make_call(api_call) result = debox_signed_syftapicall_response(signed_result=signed_result) + # relative + from ..store.blob_storage import BlobRetrievalByURL + + # In the case of blob storage, the gateway downloads the result and then passes it to + # the proxy client + if isinstance(result, BlobRetrievalByURL): + blob_route = client.api.connection.to_blob_route( + result.url.url_path + ) + result.url = blob_route + final_res = result.read() + return final_res + return result return SyftError(message=(f"Node has no route to {node_uid}")) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index b833fcbf09b..1dbf2cb28f3 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -32,6 +32,7 @@ from ...serde.serializable import serializable from ...serde.serialize import _serialize as serialize from ...service.response import SyftError +from ...store.blob_storage import BlobRetrievalByURL from ...store.linked_obj import LinkedObject from ...types.blob_storage import CreateBlobStorageEntry from ...types.datetime import DateTime @@ -511,8 +512,18 @@ def reload_cache(self): blob_retrieval_object, ) return blob_retrieval_object - self.syft_action_data_cache = blob_retrieval_object.read() - self.syft_action_data_type = type(self.syft_action_data) + + if isinstance(blob_retrieval_object, ActionObject): + # In the case of gateway, we directly receive an action object + self.syft_action_data_cache = blob_retrieval_object + self.syft_action_data_type = type(self.syft_action_data) + elif isinstance(blob_retrieval_object, BlobRetrievalByURL): + self.syft_action_data_cache = blob_retrieval_object.read() + self.syft_action_data_type = type(self.syft_action_data) + else: + return SyftError( + f"Failed to retrieve object from blob storage, got {blob_retrieval_object}" + ) def _save_to_blob_storage_(self, data: Any) -> None: if not isinstance(data, ActionDataEmpty): From 5911f741b36a665946cb0f36bcf2e7ac3c0ecd43 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 29 Nov 2023 12:37:39 +0530 Subject: [PATCH 5/8] modified blob retrieval --- .../src/syft/service/action/action_object.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 1dbf2cb28f3..b0240b6788c 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -32,6 +32,7 @@ from ...serde.serializable import serializable from ...serde.serialize import _serialize as serialize from ...service.response import SyftError +from ...store.blob_storage import BlobRetrieval from ...store.blob_storage import BlobRetrievalByURL from ...store.linked_obj import LinkedObject from ...types.blob_storage import CreateBlobStorageEntry @@ -513,17 +514,20 @@ def reload_cache(self): ) return blob_retrieval_object - if isinstance(blob_retrieval_object, ActionObject): - # In the case of gateway, we directly receive an action object - self.syft_action_data_cache = blob_retrieval_object - self.syft_action_data_type = type(self.syft_action_data) - elif isinstance(blob_retrieval_object, BlobRetrievalByURL): + if isinstance(blob_retrieval_object, SyftError): + raise SyftException( + message=f"Failed to retrieve object from blob storage: {blob_retrieval_object.message}" + ) + + elif isinstance( + blob_retrieval_object, (BlobRetrievalByURL, BlobRetrieval) + ): self.syft_action_data_cache = blob_retrieval_object.read() self.syft_action_data_type = type(self.syft_action_data) else: - return SyftError( - f"Failed to retrieve object from blob storage, got {blob_retrieval_object}" - ) + # In the case of gateway, we directly receive the actual object + self.syft_action_data_cache = blob_retrieval_object + self.syft_action_data_type = type(self.syft_action_data) def _save_to_blob_storage_(self, data: Any) -> None: if not isinstance(data, ActionDataEmpty): From 3afd4911528a78df66bd67c8b60584bf1f83d9eb Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 29 Nov 2023 14:13:34 +0530 Subject: [PATCH 6/8] added test for gateway user code --- tests/integration/network/gateway_test.py | 74 +++++++++++++++++++++++ tox.ini | 4 +- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index e5c1e4e6adf..81eb28a99c3 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -1,9 +1,17 @@ +# stdlib +from textwrap import dedent +import uuid + +# third party +import numpy as np + # syft absolute import syft as sy from syft.abstract_node import NodeType from syft.client.domain_client import DomainClient from syft.client.gateway_client import GatewayClient from syft.service.network.node_peer import NodePeer +from syft.service.request.request import Request from syft.service.response import SyftSuccess from syft.service.user.user_roles import ServiceRole @@ -52,3 +60,69 @@ def test_domain_connect_to_gateway(domain_1_port, gateway_port): assert ( proxy_domain_client.api.endpoints.keys() == domain_client.api.endpoints.keys() ) + + +def random_hash() -> str: + return uuid.uuid4().hex[:16] + + +def test_domain_gateway_user_code(domain_1_port, gateway_port): + gateway_client: GatewayClient = sy.login_as_guest(port=gateway_port) + + domain_client: DomainClient = sy.login( + port=domain_1_port, email="info@openmined.org", password="changethis" + ) + + input_data = np.array([1, 2, 3]) + mock_data = np.array([4, 5, 6]) + + asset_name = random_hash() + asset = sy.Asset(name=asset_name, data=input_data, mock=mock_data) + dataset_name = random_hash() + dataset = sy.Dataset(name=dataset_name, asset_list=[asset]) + + dataset_res = domain_client.upload_dataset(dataset) + + assert isinstance(dataset_res, SyftSuccess) + + user_create_res = domain_client.register( + name="Sheldon Cooper", + email="sheldon@caltech.edu", + password="changethis", + password_verify="changethis", + institution="Caltech", + website="https://www.caltech.edu/", + ) + + assert isinstance(user_create_res, SyftSuccess) + + gateway_con_res = domain_client.connect_to_gateway(gateway_client) + assert isinstance(gateway_con_res, SyftSuccess) + + proxy_client = gateway_client.domains[0] + + proxy_ds = proxy_client.login( + email="sheldon@caltech.edu", password="changethis", password_verify="changethis" + ) + + asset = proxy_ds.datasets[0].assets[0] + + @sy.syft_function_single_use(asset=asset) + def test_function(asset): + return asset + 1 + + test_function.code = dedent(test_function.code) + + request_res = proxy_ds.code.request_code_execution(test_function) + assert isinstance(request_res, Request) + + assert len(domain_client.requests.get_all()) == 1 + + req_approve_res = domain_client.requests[-1].approve() + assert isinstance(req_approve_res, SyftSuccess) + + result = proxy_ds.code.test_function(asset=asset) + + final_result = result.get() + + assert (final_result == input_data + 1).all() diff --git a/tox.ini b/tox.ini index bdcb9212c8e..35db887ac11 100644 --- a/tox.ini +++ b/tox.ini @@ -793,7 +793,9 @@ commands = pytest --nbmake notebooks/api/0.8 -p no:randomly -vvvv #Integration + Gateway Connection Tests - pytest tests/integration/network -p no:randomly -vvvv + # Gateway tests are not run in kuberetes, as currently,it does not have a way to configure + # high/low side warning flag. + pytest tests/integration/network -k "not test_domain_gateway_user_code" -p no:randomly -vvvv # deleting clusters created bash -c "k3d cluster delete test-gateway-1 || true" From 87c2676d16584d461543499a0debbaf2ea51ff70 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 29 Nov 2023 14:16:08 +0530 Subject: [PATCH 7/8] add TODO comments for gateway-seaweedfs fix --- packages/syft/src/syft/service/action/action_object.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index b0240b6788c..7eedb65b6e4 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -33,7 +33,6 @@ from ...serde.serialize import _serialize as serialize from ...service.response import SyftError from ...store.blob_storage import BlobRetrieval -from ...store.blob_storage import BlobRetrievalByURL from ...store.linked_obj import LinkedObject from ...types.blob_storage import CreateBlobStorageEntry from ...types.datetime import DateTime @@ -519,13 +518,13 @@ def reload_cache(self): message=f"Failed to retrieve object from blob storage: {blob_retrieval_object.message}" ) - elif isinstance( - blob_retrieval_object, (BlobRetrievalByURL, BlobRetrieval) - ): + elif isinstance(blob_retrieval_object, (BlobRetrieval)): self.syft_action_data_cache = blob_retrieval_object.read() self.syft_action_data_type = type(self.syft_action_data) else: # In the case of gateway, we directly receive the actual object + # TODO: The ideal solution would be to stream the object from the domain through the gateway + # Currently , we are just passing the object as it is, which would be fixed later. self.syft_action_data_cache = blob_retrieval_object self.syft_action_data_type = type(self.syft_action_data) From 3774451f3ef40aef40a75dd6ff2cbc34c8e7ea9c Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 29 Nov 2023 14:28:18 +0530 Subject: [PATCH 8/8] updated blob storage tech debt comment --- packages/syft/src/syft/service/action/action_object.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 7eedb65b6e4..94ef6f66d12 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -518,7 +518,8 @@ def reload_cache(self): message=f"Failed to retrieve object from blob storage: {blob_retrieval_object.message}" ) - elif isinstance(blob_retrieval_object, (BlobRetrieval)): + elif isinstance(blob_retrieval_object, BlobRetrieval): + # TODO: This change is temporary to for gateway to be compatible with the new blob storage self.syft_action_data_cache = blob_retrieval_object.read() self.syft_action_data_type = type(self.syft_action_data) else: