From abc7f09765c2d0f87003297b90ea3d22ecd9c2c3 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Mon, 26 Feb 2024 14:51:52 +1000 Subject: [PATCH] Adding initial admin api interface --- packages/syft/setup.cfg | 1 + packages/syft/src/syft/__init__.py | 1 + packages/syft/src/syft/client/api.py | 22 ++++ packages/syft/src/syft/node/node.py | 3 + .../src/syft/protocol/protocol_version.json | 11 ++ packages/syft/src/syft/service/api/api.py | 65 +++++++++++ .../syft/src/syft/service/api/api_service.py | 110 ++++++++++++++++++ 7 files changed, 213 insertions(+) create mode 100644 packages/syft/src/syft/service/api/api.py create mode 100644 packages/syft/src/syft/service/api/api_service.py diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 1a0afd7d28c..ca9a89a53a9 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -65,6 +65,7 @@ syft = kr8s==0.13.1 PyYAML==6.0.1 azure-storage-blob==12.19 + google-cloud-aiplatform install_requires = %(syft)s diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index d438f9ef9bd..d42608096e2 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -47,6 +47,7 @@ from .service.action.action_object import ActionObject # noqa: F401 from .service.action.plan import Plan # noqa: F401 from .service.action.plan import planify # noqa: F401 +from .service.api.api import api_endpoint # noqa: F401 from .service.code.user_code import UserCodeStatus # noqa: F401; noqa: F401 from .service.code.user_code import syft_function # noqa: F401; noqa: F401 from .service.code.user_code import syft_function_single_use # noqa: F401; noqa: F401 diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index 2e7c7fc25e3..8cd9df9fab4 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -620,6 +620,8 @@ def for_user( user_verify_key: Optional[SyftVerifyKey] = None, ) -> SyftAPI: # relative + from ..service.api.api_service import APIService + # TODO: Maybe there is a possibility of merging ServiceConfig and APIEndpoint from ..service.code.user_code_service import UserCodeService @@ -716,6 +718,26 @@ def for_user( ) endpoints[unique_path] = endpoint + # get admin defined custom api endpoints + method = node.get_method_with_context(APIService.get_endpoints, context) + custom_endpoints = method() + for custom_endpoint in custom_endpoints: + pre_kwargs = {"path": custom_endpoint.path} + service_path = "api.call" + path = custom_endpoint.path + api_end = custom_endpoint.path.split(".")[-1] + endpoint = APIEndpoint( + service_path=service_path, + module_path=path, + name=api_end, + description="", + doc_string="", + signature=custom_endpoint.signature, + has_self=False, + pre_kwargs=pre_kwargs, + ) + endpoints[path] = endpoint + return SyftAPI( node_name=node.name, node_uid=node.id, diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index a9b161bc63f..3913a71ac11 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -49,6 +49,7 @@ from ..service.action.action_store import DictActionStore from ..service.action.action_store import MongoActionStore from ..service.action.action_store import SQLiteActionStore +from ..service.api.api_service import APIService from ..service.blob_storage.service import BlobStorageService from ..service.code.user_code_service import UserCodeService from ..service.code.user_code_stash import UserCodeStash @@ -361,6 +362,7 @@ def __init__( SyftWorkerImageService, SyftWorkerPoolService, SyftImageRegistryService, + APIService, ] if services is None else services @@ -943,6 +945,7 @@ def _construct_services(self): SyftWorkerImageService, SyftWorkerPoolService, SyftImageRegistryService, + APIService, ] if OBLV: diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 814140d8d98..a7ca60d321d 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -7,5 +7,16 @@ }, "3": { "release_name": "0.8.4.json" + }, + "dev": { + "object_versions": { + "CustomAPIEndpoint": { + "1": { + "version": 1, + "hash": "2a85e44b93c3b41f2d7e8b3d968dd623dbf98728e26de5631c5ac22111100799", + "action": "add" + } + } + } } } diff --git a/packages/syft/src/syft/service/api/api.py b/packages/syft/src/syft/service/api/api.py new file mode 100644 index 00000000000..0c18d8a2356 --- /dev/null +++ b/packages/syft/src/syft/service/api/api.py @@ -0,0 +1,65 @@ +# stdlib +import ast +import inspect +from inspect import Signature +from typing import Any +from typing import Callable + +# relative +from ...serde.serializable import serializable +from ...serde.signature import signature_remove_context +from ...types.syft_object import SYFT_OBJECT_VERSION_1 +from ...types.syft_object import SyftObject +from ..context import AuthedServiceContext +from ..response import SyftError + + +@serializable() +class CustomAPIEndpoint(SyftObject): + # version + __canonical_name__ = "CustomAPIEndpoint" + __version__ = SYFT_OBJECT_VERSION_1 + + path: str + api_code: str + signature: Signature + func_name: str + + __attr_searchable__ = ["path"] + __attr_unique__ = ["path"] + + def exec(self, context: AuthedServiceContext, **kwargs: Any) -> Any: + try: + inner_function = ast.parse(self.api_code).body[0] + inner_function.decorator_list = [] + # compile the function + raw_byte_code = compile(ast.unparse(inner_function), "", "exec") + # load it + exec(raw_byte_code) # nosec + # execute it + evil_string = f"{self.func_name}(context, **kwargs)" + result = eval(evil_string, None, locals()) # nosec + # return the results + return context, result + except Exception as e: + print(f"Failed to run CustomAPIEndpoint Code. {e}") + return SyftError(e) + + +def get_signature(func: Callable) -> Signature: + sig = inspect.signature(func) + sig = signature_remove_context(sig) + return sig + + +def api_endpoint(path: str) -> CustomAPIEndpoint: + def decorator(f): + res = CustomAPIEndpoint( + path=path, + api_code=inspect.getsource(f), + signature=get_signature(f), + func_name=f.__name__, + ) + return res + + return decorator diff --git a/packages/syft/src/syft/service/api/api_service.py b/packages/syft/src/syft/service/api/api_service.py new file mode 100644 index 00000000000..4ccfaa862cb --- /dev/null +++ b/packages/syft/src/syft/service/api/api_service.py @@ -0,0 +1,110 @@ +# stdlib +from typing import Any +from typing import List +from typing import Union + +# third party +from result import Ok +from result import Result + +# relative +from ...node.credentials import SyftVerifyKey +from ...serde.serializable import serializable +from ...store.document_store import BaseUIDStoreStash +from ...store.document_store import DocumentStore +from ...store.document_store import PartitionSettings +from ...util.telemetry import instrument +from ..context import AuthedServiceContext +from ..response import SyftError +from ..response import SyftSuccess +from ..service import AbstractService +from ..service import service_method +from ..user.user_roles import GUEST_ROLE_LEVEL +from .api import CustomAPIEndpoint + + +@serializable() +class CustomAPIEndpointStash(BaseUIDStoreStash): + object_type = CustomAPIEndpoint + settings: PartitionSettings = PartitionSettings( + name=CustomAPIEndpoint.__canonical_name__, object_type=CustomAPIEndpoint + ) + + def __init__(self, store: DocumentStore) -> None: + super().__init__(store=store) + + def get_by_path( + self, credentials: SyftVerifyKey, path: str + ) -> Result[List[CustomAPIEndpoint], str]: + results = self.get_all(credentials=credentials) + items = [] + if results.is_ok() and results.ok(): + results = results.ok() + for result in results: + if result.path == path: + items.append(result) + return Ok(items) + else: + return results + + def update( + self, credentials: SyftVerifyKey, endpoint: CustomAPIEndpoint + ) -> Result[CustomAPIEndpoint, str]: + res = self.check_type(endpoint, CustomAPIEndpoint) + if res.is_err(): + return res + result = super().set( + credentials=credentials, obj=res.ok(), ignore_duplicates=True + ) + return result + + +@instrument +@serializable() +class APIService(AbstractService): + store: DocumentStore + stash: CustomAPIEndpointStash + + def __init__(self, store: DocumentStore) -> None: + self.store = store + self.stash = CustomAPIEndpointStash(store=store) + + @service_method(path="api.set", name="set") + def set( + self, context: AuthedServiceContext, endpoint: CustomAPIEndpoint + ) -> Union[SyftSuccess, SyftError]: + """Register an CustomAPIEndpoint.""" + result = self.stash.update(context.credentials, endpoint=endpoint) + if result.is_ok(): + return SyftSuccess(message=f"CustomAPIEndpoint added: {endpoint}") + return SyftError( + message=f"Failed to add CustomAPIEndpoint {endpoint}. {result.err()}" + ) + + def get_endpoints( + self, context: AuthedServiceContext + ) -> Union[List[CustomAPIEndpoint], SyftError]: + # TODO: Add ability to specify which roles see which endpoints + # for now skip auth + results = self.stash.get_all(context.node.verify_key) + if results.is_ok(): + return results.ok() + return SyftError(messages="Unable to get CustomAPIEndpoint") + + @service_method(path="api.call", name="call", roles=GUEST_ROLE_LEVEL) + def call( + self, + context: AuthedServiceContext, + path: str, + *args: Any, + **kwargs: Any, + ) -> Union[SyftSuccess, SyftError]: + """Call a Custom API Method""" + result = self.stash.get_by_path(context.node.verify_key, path=path) + if not result.is_ok(): + return SyftError(message=f"CustomAPIEndpoint: {path} does not exist") + custom_endpoint = result.ok() + custom_endpoint = custom_endpoint[-1] + if result: + context, result = custom_endpoint.exec(context, **kwargs) + return result