From af009b4831b6db97850008fdaed4eec7bb8ff5d9 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 25 Jun 2024 17:01:26 +0530 Subject: [PATCH 1/6] remove old enclave clode - iteration 1 --- notebooks/api/0.8/05-custom-policy.ipynb | 20 +- .../Enclave-single-notebook-DO-DS.ipynb | 735 ------------------ .../syft/src/syft/client/enclave_client.py | 52 -- .../syft/service/enclave/enclave_service.py | 149 ---- .../syft/src/syft/service/policy/policy.py | 13 - 5 files changed, 1 insertion(+), 968 deletions(-) delete mode 100644 notebooks/tutorials/enclaves/Enclave-single-notebook-DO-DS.ipynb diff --git a/notebooks/api/0.8/05-custom-policy.ipynb b/notebooks/api/0.8/05-custom-policy.ipynb index c87f5f84036..663468260b4 100644 --- a/notebooks/api/0.8/05-custom-policy.ipynb +++ b/notebooks/api/0.8/05-custom-policy.ipynb @@ -286,7 +286,7 @@ " root_context = AuthedServiceContext(\n", " node=context.node, credentials=context.node.verify_key\n", " )\n", - " if context.node.node_type == NodeType.DOMAIN:\n", + " if context.node.node_type in NodeType.DOMAIN:\n", " for var_name, arg_id in allowed_inputs.items():\n", " kwarg_value = action_service._get(\n", " context=root_context,\n", @@ -297,14 +297,6 @@ " if kwarg_value.is_err():\n", " return Err(kwarg_value.err())\n", " code_inputs[var_name] = kwarg_value.ok()\n", - "\n", - " elif context.node.node_type == NodeType.ENCLAVE:\n", - " dict_object = action_service.get(context=root_context, uid=code_item_id)\n", - " if dict_object.is_err():\n", - " return Err(dict_object.err())\n", - " for value in dict_object.ok().syft_action_data.values():\n", - " code_inputs.update(value)\n", - "\n", " else:\n", " raise Exception(\n", " f\"Invalid Node Type for Code Submission:{context.node.node_type}\"\n", @@ -328,11 +320,6 @@ " verify_key=context.node.signing_key.verify_key,\n", " )\n", " allowed_inputs = allowed_inputs.get(node_identity, {})\n", - " elif context.node.node_type == NodeType.ENCLAVE:\n", - " base_dict = {}\n", - " for key in allowed_inputs.values():\n", - " base_dict.update(key)\n", - " allowed_inputs = base_dict\n", " else:\n", " raise Exception(\n", " f\"Invalid Node Type for Code Submission:{context.node.node_type}\"\n", @@ -403,11 +390,6 @@ " verify_key=context.node.signing_key.verify_key,\n", " )\n", " allowed_inputs = allowed_inputs.get(node_identity, {})\n", - " elif context.node.node_type == NodeType.ENCLAVE:\n", - " base_dict = {}\n", - " for key in allowed_inputs.values():\n", - " base_dict.update(key)\n", - " allowed_inputs = base_dict\n", " else:\n", " raise Exception(\n", " f\"Invalid Node Type for Code Submission:{context.node.node_type}\"\n", diff --git a/notebooks/tutorials/enclaves/Enclave-single-notebook-DO-DS.ipynb b/notebooks/tutorials/enclaves/Enclave-single-notebook-DO-DS.ipynb deleted file mode 100644 index 62d7fc48c99..00000000000 --- a/notebooks/tutorials/enclaves/Enclave-single-notebook-DO-DS.ipynb +++ /dev/null @@ -1,735 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "from recordlinkage.datasets import load_febrl4\n", - "\n", - "# syft absolute\n", - "import syft as sy" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "# Create Nodes and connect to gateway" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "create enclave node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "# Local Python Node\n", - "enclave_node = sy.orchestra.launch(\n", - " name=\"Enclave\",\n", - " node_type=sy.NodeType.ENCLAVE,\n", - " local_db=True,\n", - " dev_mode=True,\n", - " reset=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.abstract_node import NodeType\n", - "\n", - "assert enclave_node.python_node.node_type == NodeType.ENCLAVE" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "Create canada node & italy node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "ca_node = sy.orchestra.launch(name=\"Canada\", local_db=True, reset=True, dev_mode=True)\n", - "it_node = sy.orchestra.launch(name=\"Italy\", local_db=True, reset=True, dev_mode=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "assert ca_node.python_node.node_type == NodeType.DOMAIN\n", - "assert it_node.python_node.node_type == NodeType.DOMAIN" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "Create gateway Node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_node = sy.orchestra.launch(\n", - " name=\"gateway\",\n", - " node_type=sy.NodeType.GATEWAY,\n", - " local_db=True,\n", - " reset=True,\n", - " dev_mode=True,\n", - " association_request_auto_approval=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "Connect nodes to gateway" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "enclave_guest_client = enclave_node.client\n", - "ca_guest_client = ca_node.client\n", - "it_guest_client = it_node.client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.client.domain_client import DomainClient\n", - "from syft.client.enclave_client import EnclaveClient\n", - "from syft.client.gateway_client import GatewayClient\n", - "\n", - "assert isinstance(enclave_guest_client, EnclaveClient)\n", - "assert isinstance(ca_guest_client, DomainClient)\n", - "assert isinstance(it_guest_client, DomainClient)\n", - "assert isinstance(gateway_node.client, GatewayClient)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "# Connect enclave to gateway\n", - "from syft.service.response import SyftSuccess\n", - "\n", - "res = enclave_guest_client.connect_to_gateway(handle=gateway_node)\n", - "assert isinstance(res, SyftSuccess)\n", - "res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], - "source": [ - "# Connect Canada to gateway\n", - "res = ca_guest_client.connect_to_gateway(handle=gateway_node)\n", - "assert isinstance(res, SyftSuccess)\n", - "res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15", - "metadata": {}, - "outputs": [], - "source": [ - "# Connect Italy to gateway\n", - "res = it_guest_client.connect_to_gateway(handle=gateway_node)\n", - "assert isinstance(res, SyftSuccess)\n", - "res" - ] - }, - { - "cell_type": "markdown", - "id": "16", - "metadata": {}, - "source": [ - "# DOs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "17", - "metadata": {}, - "outputs": [], - "source": [ - "do_ca_client = ca_node.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "do_it_client = it_node.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.client.domain_client import DomainClient\n", - "\n", - "assert isinstance(do_ca_client, DomainClient)\n", - "assert isinstance(do_it_client, DomainClient)" - ] - }, - { - "cell_type": "markdown", - "id": "19", - "metadata": {}, - "source": [ - "## Upload dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20", - "metadata": {}, - "outputs": [], - "source": [ - "# Using public datasets from Freely Extensible Biomedical Record Linkage (Febrl)\n", - "canada_census_data, italy_census_data = load_febrl4()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21", - "metadata": {}, - "outputs": [], - "source": [ - "for dataset, client, country in zip(\n", - " [canada_census_data, italy_census_data],\n", - " [do_ca_client, do_it_client],\n", - " [\"Canada\", \"Italy\"],\n", - "):\n", - " private_data, mock_data = dataset[:2500], dataset[2500:]\n", - " dataset = sy.Dataset(\n", - " name=f\"{country} - FEBrl Census Data\",\n", - " description=\"abc\",\n", - " asset_list=[\n", - " sy.Asset(\n", - " name=\"census_data\",\n", - " mock=mock_data,\n", - " data=private_data,\n", - " shape=private_data.shape,\n", - " mock_is_real=True,\n", - " )\n", - " ],\n", - " )\n", - " client.upload_dataset(dataset)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22", - "metadata": {}, - "outputs": [], - "source": [ - "assert len(do_ca_client.datasets.get_all()) == 1\n", - "assert len(do_it_client.datasets.get_all()) == 1" - ] - }, - { - "cell_type": "markdown", - "id": "23", - "metadata": {}, - "source": [ - "## create accounts for DS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24", - "metadata": {}, - "outputs": [], - "source": [ - "for client in [do_ca_client, do_it_client]:\n", - " res = client.register(\n", - " name=\"Sheldon\",\n", - " email=\"sheldon@caltech.edu\",\n", - " password=\"changethis\",\n", - " password_verify=\"changethis\",\n", - " )\n", - " assert isinstance(res, SyftSuccess)" - ] - }, - { - "cell_type": "markdown", - "id": "25", - "metadata": {}, - "source": [ - "# DS" - ] - }, - { - "cell_type": "markdown", - "id": "26", - "metadata": {}, - "source": [ - "## Login into gateway as guest" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27", - "metadata": {}, - "outputs": [], - "source": [ - "ds_gateway_client = gateway_node.client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28", - "metadata": {}, - "outputs": [], - "source": [ - "# Explore the domains and enclaves connected to the gateway\n", - "ds_gateway_client.domains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29", - "metadata": {}, - "outputs": [], - "source": [ - "# Log into canada as proxy_client\n", - "ds_ca_proxy_client = ds_gateway_client.domains[0]\n", - "ds_ca_proxy_client = ds_ca_proxy_client.login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")\n", - "assert ds_ca_proxy_client.name == \"Canada\"\n", - "assert ds_ca_proxy_client.connection.proxy_target_uid == do_ca_client.id\n", - "assert isinstance(ds_ca_proxy_client, DomainClient)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30", - "metadata": {}, - "outputs": [], - "source": [ - "# Log into italy as proxy_client\n", - "ds_it_proxy_client = ds_gateway_client.domains[1]\n", - "ds_it_proxy_client = ds_it_proxy_client.login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")\n", - "assert ds_it_proxy_client.name == \"Italy\"\n", - "assert ds_it_proxy_client.connection.proxy_target_uid == do_it_client.id\n", - "assert isinstance(ds_it_proxy_client, DomainClient)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31", - "metadata": {}, - "outputs": [], - "source": [ - "# Create an account and log into enclave as proxy client\n", - "ds_enclave_proxy_client = ds_gateway_client.enclaves[0]\n", - "ds_enclave_proxy_client = ds_enclave_proxy_client.login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\", name=\"Sheldon\", register=True\n", - ")\n", - "assert ds_enclave_proxy_client.name == \"Enclave\"\n", - "assert ds_enclave_proxy_client.connection.proxy_target_uid == enclave_guest_client.id\n", - "assert isinstance(ds_enclave_proxy_client, EnclaveClient)" - ] - }, - { - "cell_type": "markdown", - "id": "32", - "metadata": {}, - "source": [ - "## Find datasets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33", - "metadata": {}, - "outputs": [], - "source": [ - "canada_census_data = ds_ca_proxy_client.datasets[-1].assets[0]\n", - "italy_census_data = ds_it_proxy_client.datasets[-1].assets[0]" - ] - }, - { - "cell_type": "markdown", - "id": "34", - "metadata": {}, - "source": [ - "## Create Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35", - "metadata": {}, - "outputs": [], - "source": [ - "@sy.syft_function_single_use(\n", - " canada_census_data=canada_census_data,\n", - " italy_census_data=italy_census_data,\n", - " share_results_with_owners=True,\n", - ")\n", - "def compute_census_matches(canada_census_data, italy_census_data):\n", - " # third party\n", - " import recordlinkage\n", - "\n", - " # Index step\n", - " indexer = recordlinkage.Index()\n", - " indexer.block(\"given_name\")\n", - "\n", - " candidate_links = indexer.index(canada_census_data, italy_census_data)\n", - "\n", - " # Comparison step\n", - " compare_cl = recordlinkage.Compare()\n", - "\n", - " compare_cl.exact(\"given_name\", \"given_name\", label=\"given_name\")\n", - " compare_cl.string(\n", - " \"surname\", \"surname\", method=\"jarowinkler\", threshold=0.85, label=\"surname\"\n", - " )\n", - " compare_cl.exact(\"date_of_birth\", \"date_of_birth\", label=\"date_of_birth\")\n", - " compare_cl.exact(\"suburb\", \"suburb\", label=\"suburb\")\n", - " compare_cl.exact(\"state\", \"state\", label=\"state\")\n", - " compare_cl.string(\"address_1\", \"address_1\", threshold=0.85, label=\"address_1\")\n", - "\n", - " features = compare_cl.compute(\n", - " candidate_links, canada_census_data, italy_census_data\n", - " )\n", - "\n", - " # Classification step\n", - " matches = features[features.sum(axis=1) > 3]\n", - "\n", - " return len(matches)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36", - "metadata": {}, - "outputs": [], - "source": [ - "# Check result of mock data execution\n", - "mock_result = compute_census_matches(\n", - " canada_census_data=canada_census_data.mock,\n", - " italy_census_data=italy_census_data.mock,\n", - ")\n", - "mock_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37", - "metadata": {}, - "outputs": [], - "source": [ - "req = ds_enclave_proxy_client.request_code_execution(compute_census_matches)\n", - "req" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38", - "metadata": {}, - "outputs": [], - "source": [ - "assert isinstance(req, sy.service.request.request.Request)" - ] - }, - { - "cell_type": "markdown", - "id": "39", - "metadata": {}, - "source": [ - "# DOs" - ] - }, - { - "cell_type": "markdown", - "id": "40", - "metadata": {}, - "source": [ - "## Approve" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41", - "metadata": {}, - "outputs": [], - "source": [ - "for client in [do_ca_client, do_it_client]:\n", - " res = client.requests[-1].approve()\n", - " assert isinstance(res, SyftSuccess)" - ] - }, - { - "cell_type": "markdown", - "id": "42", - "metadata": {}, - "source": [ - "# DS" - ] - }, - { - "cell_type": "markdown", - "id": "43", - "metadata": {}, - "source": [ - "## Get result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44", - "metadata": {}, - "outputs": [], - "source": [ - "status = ds_enclave_proxy_client.code.get_all()[-1].status\n", - "status" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "45", - "metadata": {}, - "outputs": [], - "source": [ - "for st, _ in status.status_dict.values():\n", - " assert st == sy.service.request.request.UserCodeStatus.APPROVED" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "46", - "metadata": {}, - "outputs": [], - "source": [ - "ds_enclave_proxy_client.code[-1].output_policy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer = ds_enclave_proxy_client.code.compute_census_matches(\n", - " canada_census_data=canada_census_data, italy_census_data=italy_census_data\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "48", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "49", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer.syft_action_data == 858" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50", - "metadata": {}, - "outputs": [], - "source": [ - "real_result = result_pointer.get()\n", - "real_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51", - "metadata": {}, - "outputs": [], - "source": [ - "assert real_result == 813" - ] - }, - { - "cell_type": "markdown", - "id": "52", - "metadata": {}, - "source": [ - "# DO" - ] - }, - { - "cell_type": "markdown", - "id": "53", - "metadata": {}, - "source": [ - "## Can also get the result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "54", - "metadata": {}, - "outputs": [], - "source": [ - "request = do_ca_client.requests[0]\n", - "request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "55", - "metadata": {}, - "outputs": [], - "source": [ - "result_ptr = request.get_results()\n", - "result_ptr" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "56", - "metadata": {}, - "outputs": [], - "source": [ - "assert result_ptr.syft_action_data == 813" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": { - "height": "calc(100% - 180px)", - "left": "10px", - "top": "150px", - "width": "358.398px" - }, - "toc_section_display": true, - "toc_window_display": true - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/packages/syft/src/syft/client/enclave_client.py b/packages/syft/src/syft/client/enclave_client.py index 32eebdf3189..6252817f630 100644 --- a/packages/syft/src/syft/client/enclave_client.py +++ b/packages/syft/src/syft/client/enclave_client.py @@ -2,12 +2,10 @@ from __future__ import annotations # stdlib -from typing import Any from typing import TYPE_CHECKING # relative from ..abstract_node import NodeSideType -from ..client.api import APIRegistry from ..serde.serializable import serializable from ..service.metadata.node_metadata import NodeMetadataJSON from ..service.network.routes import NodeRouteType @@ -15,7 +13,6 @@ from ..service.response import SyftSuccess from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.syft_object import SyftObject -from ..types.uid import UID from ..util.assets import load_png_base64 from ..util.notebook_ui.styles import FONT_CSS from .api import APIModule @@ -27,7 +24,6 @@ if TYPE_CHECKING: # relative from ..orchestra import NodeHandle - from ..service.code.user_code import SubmitUserCode @serializable() @@ -109,54 +105,6 @@ def connect_to_gateway( def get_enclave_metadata(self) -> EnclaveMetadata: return EnclaveMetadata(route=self.connection.route) - def request_code_execution(self, code: SubmitUserCode) -> Any | SyftError: - # relative - from ..service.code.user_code_service import SubmitUserCode - - if not isinstance(code, SubmitUserCode): - raise Exception( - f"The input code should be of type: {SubmitUserCode} got:{type(code)}" - ) - if code.input_policy_init_kwargs is None: - raise ValueError(f"code {code}'s input_policy_init_kwargs is None") - - enclave_metadata = self.get_enclave_metadata() - - code_id = UID() - code.id = code_id - code.enclave_metadata = enclave_metadata - - apis = [] - for k, v in code.input_policy_init_kwargs.items(): - # We would need the verify key of the data scientist to be able to index the correct client - # Since we do not want the data scientist to pass in the clients to the enclave client - # from a UX perspecitve. - # we will use the recent node id to find the correct client - # assuming that it is the correct client - # Warning: This could lead to inconsistent results, when we have multiple clients - # in the same node pointing to the same node. - # One way, by which we could solve this in the long term, - # by forcing the user to pass only assets to the sy.ExactMatch, - # by which we could extract the verify key of the data scientist - # as each object comes with a verify key and node_uid - # the asset object would contain the verify key of the data scientist. - api = APIRegistry.get_by_recent_node_uid(k.node_id) - if api is None: - raise ValueError(f"could not find client for input {v}") - else: - apis += [api] - - for api in apis: - res = api.services.code.request_code_execution(code=code) - if isinstance(res, SyftError): - return res - - # we are using the real method here, see the .code property getter - _ = self.code - res = self._request_code_execution(code=code) - - return res - def _repr_html_(self) -> str: commands = """
  • <your_client> diff --git a/packages/syft/src/syft/service/enclave/enclave_service.py b/packages/syft/src/syft/service/enclave/enclave_service.py index 46afbb6af8c..0807ada8da0 100644 --- a/packages/syft/src/syft/service/enclave/enclave_service.py +++ b/packages/syft/src/syft/service/enclave/enclave_service.py @@ -1,163 +1,14 @@ # stdlib # relative -from ...client.enclave_client import EnclaveClient -from ...client.enclave_client import EnclaveMetadata from ...serde.serializable import serializable -from ...service.response import SyftError -from ...service.response import SyftSuccess -from ...service.user.user_roles import GUEST_ROLE_LEVEL from ...store.document_store import DocumentStore -from ...types.twin_object import TwinObject -from ...types.uid import UID -from ..action.action_object import ActionObject -from ..code.user_code import UserCode -from ..code.user_code import UserCodeStatus -from ..context import AuthedServiceContext -from ..context import ChangeContext -from ..network.routes import route_to_connection -from ..policy.policy import InputPolicy from ..service import AbstractService -from ..service import service_method -# TODO 🟣 Created a generic Enclave Service -# Currently it mainly works only for Azure @serializable() class EnclaveService(AbstractService): store: DocumentStore def __init__(self, store: DocumentStore) -> None: self.store = store - - @service_method( - path="enclave.send_user_code_inputs_to_enclave", - name="send_user_code_inputs_to_enclave", - roles=GUEST_ROLE_LEVEL, - ) - def send_user_code_inputs_to_enclave( - self, - context: AuthedServiceContext, - user_code_id: UID, - inputs: dict, - node_name: str, - node_id: UID, - ) -> SyftSuccess | SyftError: - if not context.node or not context.node.signing_key: - return SyftError(message=f"{type(context)} has no node") - - root_context = AuthedServiceContext( - credentials=context.node.verify_key, node=context.node - ) - - user_code_service = context.node.get_service("usercodeservice") - action_service = context.node.get_service("actionservice") - user_code = user_code_service.get_by_uid(context=root_context, uid=user_code_id) - if isinstance(user_code, SyftError): - return user_code - - reason: str = context.extra_kwargs.get("reason", "") - status_update = user_code.get_status(root_context).mutate( - value=(UserCodeStatus.APPROVED, reason), - node_name=node_name, - node_id=node_id, - verify_key=context.credentials, - ) - if isinstance(status_update, SyftError): - return status_update - - res = user_code.status_link.update_with_context(root_context, status_update) - if isinstance(res, SyftError): - return res - - root_context = context.as_root_context() - if not action_service.exists(context=context, obj_id=user_code_id): - dict_object = ActionObject.from_obj({}) - dict_object.id = user_code_id - dict_object[str(context.credentials)] = inputs - root_context.extra_kwargs = {"has_result_read_permission": True} - # TODO: Instead of using the action store, modify to - # use the action service directly to store objects - # TODO: we store this in the actionstore isntead of blob stoarge, - # which is bad, but we cannot update in the blobstorage - res = action_service._set( - root_context, - dict_object, - ignore_detached_objs=True, - skip_clear_cache=True, - ) - if res.is_err(): - return SyftError(message=res.value) - - else: - res = action_service.get(uid=user_code_id, context=root_context) - if res.is_ok(): - dict_object = res.ok() - dict_object[str(context.credentials)] = inputs - # TODO: we store this in the actionstore isntead of blob stoarge, - # which is bad, but we cannot update in the blobstorage - res = action_service._set( - root_context, - dict_object, - ignore_detached_objs=True, - skip_clear_cache=True, - ) - if res.is_err(): - return SyftError(message=res.value) - else: - return SyftError( - message=f"Error while fetching the object on Enclave: {res.err()}" - ) - - return SyftSuccess(message="Enclave Code Status Updated Successfully") - - -# Checks if the given user code would propogate value to enclave on acceptance -def propagate_inputs_to_enclave( - user_code: UserCode, context: ChangeContext -) -> SyftSuccess | SyftError: - if isinstance(user_code.enclave_metadata, EnclaveMetadata): - # TODO 🟣 Restructure url it work for local mode host.docker.internal - - connection = route_to_connection(user_code.enclave_metadata.route) - enclave_client = EnclaveClient( - connection=connection, - credentials=context.node.signing_key, - ) - - send_method = ( - enclave_client.api.services.enclave.send_user_code_inputs_to_enclave - ) - - else: - return SyftSuccess(message="Current Request does not require Enclave Transfer") - - input_policy: InputPolicy | None = user_code.get_input_policy( - context.to_service_ctx() - ) - if input_policy is None: - return SyftError(message=f"{user_code}'s input policy is None") - inputs = input_policy._inputs_for_context(context) - if isinstance(inputs, SyftError): - return inputs - - # Save inputs to blob store - for var_name, var_value in inputs.items(): - if isinstance(var_value, ActionObject | TwinObject): - # Set the obj location to enclave - var_value._set_obj_location_( - enclave_client.api.node_uid, - enclave_client.verify_key, - ) - var_value._save_to_blob_storage() - - inputs[var_name] = var_value - - # send data of the current node to enclave - res = send_method( - user_code_id=user_code.id, - inputs=inputs, - node_name=context.node.name, - node_id=context.node.id, - ) - return res diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index 0e4c791d3ea..fa97790ba39 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -594,14 +594,6 @@ def retrieve_from_db( if kwarg_value.is_err(): return Err(kwarg_value.err()) code_inputs[var_name] = kwarg_value.ok() - - elif context.node.node_type == NodeType.ENCLAVE: - dict_object = action_service.get(context=root_context, uid=code_item_id) - if dict_object.is_err(): - return Err(dict_object.err()) - for value in dict_object.ok().syft_action_data.values(): - code_inputs.update(value) - else: raise Exception( f"Invalid Node Type for Code Submission:{context.node.node_type}" @@ -621,11 +613,6 @@ def allowed_ids_only( verify_key=context.node.signing_key.verify_key, ) allowed_inputs = allowed_inputs.get(node_identity, {}) - elif context.node.node_type == NodeType.ENCLAVE: - base_dict = {} - for key in allowed_inputs.values(): - base_dict.update(key) - allowed_inputs = base_dict else: raise Exception( f"Invalid Node Type for Code Submission:{context.node.node_type}" From 3bfdb366a2ee8392746ca3dabe6ec6f488abff92 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 25 Jun 2024 19:27:23 +0530 Subject: [PATCH 2/6] remove enclave code in request.py --- .../syft/src/syft/service/request/request.py | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 011e96c25d7..b2732c69f5e 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -670,7 +670,7 @@ def approve( metadata = api.connection.get_node_metadata(api.signing_key) else: metadata = None - message, is_enclave = None, False + message = None is_code_request = not isinstance(self.codes, SyftError) @@ -679,12 +679,7 @@ def approve( message="Multiple codes detected, please use approve_nested=True" ) - if self.code and not isinstance(self.code, SyftError): - is_enclave = getattr(self.code, "enclave_metadata", None) is not None - - if is_enclave: - message = "On approval, the result will be released to the enclave." - elif metadata and metadata.node_side_type == NodeSideType.HIGH_SIDE.value: + if metadata and metadata.node_side_type == NodeSideType.HIGH_SIDE.value: message = ( "You're approving a request on " f"{metadata.node_side_type} side {metadata.node_type} " @@ -1317,12 +1312,6 @@ def mutate( ) return res - def is_enclave_request(self, user_code: UserCode) -> bool: - return ( - user_code.is_enclave_code is not None - and self.value == UserCodeStatus.APPROVED - ) - def _run( self, context: ChangeContext, apply: bool ) -> Result[SyftSuccess, SyftError]: @@ -1346,16 +1335,7 @@ def _run( if isinstance(updated_status, SyftError): return Err(updated_status.message) - # relative - from ..enclave.enclave_service import propagate_inputs_to_enclave - self.linked_obj.update_with_context(context, updated_status) - if self.is_enclave_request(user_code): - enclave_res = propagate_inputs_to_enclave( - user_code=user_code, context=context - ) - if isinstance(enclave_res, SyftError): - return enclave_res else: updated_status = self.mutate(user_code_status, context, undo=True) if isinstance(updated_status, SyftError): From 72b6a444c20ab3dcf99cd7c3b672c5d9aec5b61c Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 25 Jun 2024 19:37:56 +0530 Subject: [PATCH 3/6] remove enclave code in request and user code --- .../syft/src/syft/service/code/user_code.py | 53 +++++++++++------ .../syft/service/code/user_code_service.py | 59 ------------------- .../syft/service/code_history/code_history.py | 17 +++++- .../syft/src/syft/service/request/request.py | 4 -- 4 files changed, 52 insertions(+), 81 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index b71f5aa4cc6..a3330161a7c 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -54,6 +54,7 @@ from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import SYFT_OBJECT_VERSION_4 from ...types.syft_object import SYFT_OBJECT_VERSION_5 +from ...types.syft_object import SYFT_OBJECT_VERSION_6 from ...types.syft_object import SyftObject from ...types.syncable_object import SyncableSyftObject from ...types.transforms import TransformContext @@ -299,7 +300,7 @@ class UserCodeV4(SyncableSyftObject): @serializable() -class UserCode(SyncableSyftObject): +class UserCodeV5(SyncableSyftObject): # version __canonical_name__ = "UserCode" __version__ = SYFT_OBJECT_VERSION_5 @@ -332,6 +333,40 @@ class UserCode(SyncableSyftObject): origin_node_side_type: NodeSideType l0_deny_reason: str | None = None + +@serializable() +class UserCode(SyncableSyftObject): + # version + __canonical_name__ = "UserCode" + __version__ = SYFT_OBJECT_VERSION_6 + + id: UID + node_uid: UID | None = None + user_verify_key: SyftVerifyKey + raw_code: str + input_policy_type: type[InputPolicy] | UserPolicy + input_policy_init_kwargs: dict[Any, Any] | None = None + input_policy_state: bytes = b"" + output_policy_type: type[OutputPolicy] | UserPolicy + output_policy_init_kwargs: dict[Any, Any] | None = None + output_policy_state: bytes = b"" + parsed_code: str + service_func_name: str + unique_func_name: str + user_unique_func_name: str + code_hash: str + signature: inspect.Signature + status_link: LinkedObject | None = None + input_kwargs: list[str] + submit_time: DateTime | None = None + # tracks if the code calls domain.something, variable is set during parsing + uses_domain: bool = False + + nested_codes: dict[str, tuple[LinkedObject, dict]] | None = {} + worker_pool_name: str | None = None + origin_node_side_type: NodeSideType + l0_deny_reason: str | None = None + __table_coll_widths__ = [ "min-content", "auto", @@ -510,10 +545,6 @@ def get_status( return SyftError(message=status.err()) return status.ok() - @property - def is_enclave_code(self) -> bool: - return self.enclave_metadata is not None - @property def input_owners(self) -> list[str] | None: if self.input_policy_init_kwargs is not None: @@ -738,17 +769,6 @@ def store_execution_output( def byte_code(self) -> PyCodeObject | None: return compile_byte_code(self.parsed_code) - def get_results(self) -> Any: - # relative - from ...client.api import APIRegistry - - api = APIRegistry.api_for(self.node_uid, self.syft_client_verify_key) - if api is None: - return SyftError( - message=f"Can't access the api. You must login to {self.node_uid}" - ) - return api.services.code.get_results(self) - @property def assets(self) -> list[Asset]: # relative @@ -926,7 +946,6 @@ class SubmitUserCode(SyftObject): output_policy_init_kwargs: dict[Any, Any] | None = {} local_function: Callable | None = None input_kwargs: list[str] - enclave_metadata: EnclaveMetadata | None = None worker_pool_name: str | None = None __repr_attrs__ = ["func_name", "code"] diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 58ec982ac2c..374fcf22475 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -9,8 +9,6 @@ from result import Result # relative -from ...abstract_node import NodeType -from ...client.enclave_client import EnclaveClient from ...serde.serializable import serializable from ...store.document_store import DocumentStore from ...store.linked_obj import LinkedObject @@ -23,7 +21,6 @@ from ..action.action_permissions import ActionObjectPermission from ..action.action_permissions import ActionPermission from ..context import AuthedServiceContext -from ..network.routes import route_to_connection from ..output.output_service import ExecutionOutput from ..policy.policy import OutputPolicy from ..request.request import Request @@ -316,62 +313,6 @@ def load_user_code(self, context: AuthedServiceContext) -> None: user_code_items = result.ok() load_approved_policy_code(user_code_items=user_code_items, context=context) - @service_method(path="code.get_results", name="get_results", roles=GUEST_ROLE_LEVEL) - def get_results( - self, context: AuthedServiceContext, inp: UID | UserCode - ) -> list[UserCode] | SyftError: - uid = inp.id if isinstance(inp, UserCode) else inp - code_result = self.stash.get_by_uid(context.credentials, uid=uid) - - if code_result.is_err(): - return SyftError(message=code_result.err()) - code = code_result.ok() - - if code.is_enclave_code: - # if the current node is not the enclave - if not context.node.node_type == NodeType.ENCLAVE: - connection = route_to_connection(code.enclave_metadata.route) - enclave_client = EnclaveClient( - connection=connection, - credentials=context.node.signing_key, - ) - if enclave_client.code is None: - return SyftError( - message=f"{enclave_client} can't access the user code api" - ) - outputs = enclave_client.code.get_results(code.id) - if isinstance(outputs, list): - for output in outputs: - output.syft_action_data # noqa: B018 - else: - outputs.syft_action_data # noqa: B018 - return outputs - - # if the current node is the enclave - else: - if not code.get_status(context.as_root_context()).approved: - return code.status.get_status_message() - - output_history = code.get_output_history( - context=context.as_root_context() - ) - if isinstance(output_history, SyftError): - return output_history - - if len(output_history) > 0: - res = resolve_outputs( - context=context, - output_ids=output_history[-1].output_ids, - ) - if res.is_err(): - return res - res = delist_if_single(res.ok()) - return Ok(res) - else: - return SyftError(message="No results available") - else: - return SyftError(message="Endpoint only supported for enclave code") - def is_execution_allowed( self, code: UserCode, diff --git a/packages/syft/src/syft/service/code_history/code_history.py b/packages/syft/src/syft/service/code_history/code_history.py index c3b1151fe2e..488083cf0c6 100644 --- a/packages/syft/src/syft/service/code_history/code_history.py +++ b/packages/syft/src/syft/service/code_history/code_history.py @@ -8,6 +8,7 @@ from ...serde.serializable import serializable from ...service.user.user_roles import ServiceRole from ...types.syft_object import SYFT_OBJECT_VERSION_2 +from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...types.syft_object import SyftObject from ...types.syft_object import SyftVerifyKey from ...types.uid import UID @@ -20,7 +21,7 @@ @serializable() -class CodeHistory(SyftObject): +class CodeHistoryV2(SyftObject): # version __canonical_name__ = "CodeHistory" __version__ = SYFT_OBJECT_VERSION_2 @@ -33,6 +34,20 @@ class CodeHistory(SyftObject): service_func_name: str comment_history: list[str] = [] + +@serializable() +class CodeHistory(SyftObject): + # version + __canonical_name__ = "CodeHistory" + __version__ = SYFT_OBJECT_VERSION_3 + + id: UID + node_uid: UID + user_verify_key: SyftVerifyKey + user_code_history: list[UID] = [] + service_func_name: str + comment_history: list[str] = [] + __attr_searchable__ = ["user_verify_key", "service_func_name"] def add_code(self, code: UserCode, comment: str | None = None) -> None: diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index b2732c69f5e..2680bc31b14 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -610,9 +610,6 @@ def code(self) -> UserCode | SyftError: message="This type of request does not have code associated with it." ) - def get_results(self) -> Any: - return self.code.get_results() - @property def current_change_state(self) -> dict[UID, bool]: change_applied_map = {} @@ -1341,7 +1338,6 @@ def _run( if isinstance(updated_status, SyftError): return Err(updated_status.message) - # TODO: Handle Enclave approval. self.linked_obj.update_with_context(context, updated_status) return Ok(SyftSuccess(message=f"{type(self)} Success")) except Exception as e: From 46320181e9bef52ffe44c5ad349276d47179cd6d Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 25 Jun 2024 19:40:33 +0530 Subject: [PATCH 4/6] fix mypy --- packages/syft/src/syft/service/policy/policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index fa97790ba39..e4800f04d6e 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -605,7 +605,7 @@ def allowed_ids_only( allowed_inputs: dict[NodeIdentity, Any], kwargs: dict[str, Any], context: AuthedServiceContext, -) -> dict[str, UID]: +) -> dict[NodeIdentity, UID]: if context.node.node_type == NodeType.DOMAIN: node_identity = NodeIdentity( node_name=context.node.name, From 2cea46823f1420472afa074ca2230fe25de633e8 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 25 Jun 2024 19:43:04 +0530 Subject: [PATCH 5/6] fix submit user code --- .../syft/src/syft/service/code/user_code.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index a3330161a7c..cbd260a64b7 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -931,11 +931,31 @@ class UserCodeUpdate(PartialSyftObject): @serializable(without=["local_function"]) -class SubmitUserCode(SyftObject): +class SubmitUserCodeV4(SyftObject): # version __canonical_name__ = "SubmitUserCode" __version__ = SYFT_OBJECT_VERSION_4 + id: UID | None = None # type: ignore[assignment] + code: str + func_name: str + signature: inspect.Signature + input_policy_type: SubmitUserPolicy | UID | type[InputPolicy] + input_policy_init_kwargs: dict[Any, Any] | None = {} + output_policy_type: SubmitUserPolicy | UID | type[OutputPolicy] + output_policy_init_kwargs: dict[Any, Any] | None = {} + local_function: Callable | None = None + input_kwargs: list[str] + enclave_metadata: EnclaveMetadata | None = None + worker_pool_name: str | None = None + + +@serializable(without=["local_function"]) +class SubmitUserCode(SyftObject): + # version + __canonical_name__ = "SubmitUserCode" + __version__ = SYFT_OBJECT_VERSION_5 + id: UID | None = None # type: ignore[assignment] code: str func_name: str From eb1fc6be5a5c32a6e9596c4a29c5285e2f778727 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 25 Jun 2024 19:45:23 +0530 Subject: [PATCH 6/6] fix custom policy notebook --- notebooks/api/0.8/05-custom-policy.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/api/0.8/05-custom-policy.ipynb b/notebooks/api/0.8/05-custom-policy.ipynb index 663468260b4..8c7b18c9328 100644 --- a/notebooks/api/0.8/05-custom-policy.ipynb +++ b/notebooks/api/0.8/05-custom-policy.ipynb @@ -286,7 +286,7 @@ " root_context = AuthedServiceContext(\n", " node=context.node, credentials=context.node.verify_key\n", " )\n", - " if context.node.node_type in NodeType.DOMAIN:\n", + " if context.node.node_type == NodeType.DOMAIN:\n", " for var_name, arg_id in allowed_inputs.items():\n", " kwarg_value = action_service._get(\n", " context=root_context,\n",