From 501d42d154c62c9940f66c0f937793942375e3d7 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Wed, 3 Aug 2022 12:57:15 +0530 Subject: [PATCH 1/3] new API endpoint added to get persistent plugins Signed-off-by: ashish-jabble --- python/fledge/plugins/common/utils.py | 26 +++++++++++++++++-- .../fledge/services/core/api/plugins/data.py | 15 +++++++++++ python/fledge/services/core/routes.py | 1 + 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/python/fledge/plugins/common/utils.py b/python/fledge/plugins/common/utils.py index 101289fb0..41c57df81 100644 --- a/python/fledge/plugins/common/utils.py +++ b/python/fledge/plugins/common/utils.py @@ -8,13 +8,16 @@ import datetime -__author__ = "Amarendra Kumar Sinha" +from fledge.services.core.api import utils as api_utils + +__author__ = "Amarendra Kumar Sinha, Ashish Jabble" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" __license__ = "Apache 2.0" __version__ = "${VERSION}" DEPRECATED_BIT_POSITION = 7 DEPRECATED_BIT_MASK_VALUE = 1 << DEPRECATED_BIT_POSITION +PERSIST_DATA_BIT_POSITION = 3 def get_diff(old, new): @@ -31,7 +34,7 @@ def get_diff(old, new): def local_timestamp(): """ :return: str - current time stamp with microseconds and machine timezone info - :example '2018-05-08 14:06:40.517313+05:30' + :example: '2018-05-08 14:06:40.51731305:30' """ return str(datetime.datetime.now(datetime.timezone.utc).astimezone()) @@ -51,3 +54,22 @@ def bit_at_given_position_set_or_unset(n, k): """ new_num = n >> k return new_num & 1 + + +def get_persist_plugins(): + """ Get a list of south, north, filter types plugins that can persist data for a service + :return: list - plugins + :example: ["OMF"] + """ + plugin_list = [] + supported_persist_dirs = ["south", "north", "filter"] + for plugin_type in supported_persist_dirs: + libs = api_utils.find_c_plugin_libs(plugin_type) + for name, _type in libs: + if _type == 'binary': + jdoc = api_utils.get_plugin_info(name, dir=plugin_type) + if jdoc: + if 'flag' in jdoc: + if bit_at_given_position_set_or_unset(jdoc['flag'], PERSIST_DATA_BIT_POSITION): + plugin_list.append(jdoc['name']) + return plugin_list diff --git a/python/fledge/services/core/api/plugins/data.py b/python/fledge/services/core/api/plugins/data.py index efda506ab..ef45129b7 100644 --- a/python/fledge/services/core/api/plugins/data.py +++ b/python/fledge/services/core/api/plugins/data.py @@ -8,6 +8,7 @@ import urllib.parse from aiohttp import web +from fledge.plugins.common import utils as common_utils from fledge.common.plugin_discovery import PluginDiscovery from fledge.common.storage_client.payload_builder import PayloadBuilder from fledge.services.core import connect @@ -19,11 +20,25 @@ _help = """ --------------------------------------------------------------------------------------- + | GET | /fledge/service/{service_name}/persist | | GET POST DELETE | /fledge/service/{service_name}/plugin/{plugin_name}/data | --------------------------------------------------------------------------------------- """ +async def get_persist_plugins(request: web.Request) -> web.Response: + """ + Args: + request: + Returns: + list of plugins that have SP_PERSIST_DATA flag set in plugin info + :Example: + curl -sX GET "http://localhost:8081/fledge/service/{service_name}/persist" + """ + plugins = common_utils.get_persist_plugins() + return web.json_response({'persistent': plugins}) + + async def get(request: web.Request) -> web.Response: """ Args: diff --git a/python/fledge/services/core/routes.py b/python/fledge/services/core/routes.py index 52a5919fb..a6f2729ad 100644 --- a/python/fledge/services/core/routes.py +++ b/python/fledge/services/core/routes.py @@ -187,6 +187,7 @@ def setup(app): app.router.add_route('DELETE', '/fledge/plugins/{type}/{name}', plugins_remove.remove_plugin) # plugin data + app.router.add_route('GET', '/fledge/service/{service_name}/persist', plugin_data.get_persist_plugins) app.router.add_route('GET', '/fledge/service/{service_name}/plugin/{plugin_name}/data', plugin_data.get) app.router.add_route('POST', '/fledge/service/{service_name}/plugin/{plugin_name}/data', plugin_data.add) app.router.add_route('DELETE', '/fledge/service/{service_name}/plugin/{plugin_name}/data', plugin_data.delete) From 80017b5dfd8ca8b1e844d4ce0d49473cad7eb94f Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Wed, 3 Aug 2022 13:37:44 +0530 Subject: [PATCH 2/3] Forbidden handling added in POST plugin data and http status code fixes Signed-off-by: ashish-jabble --- python/fledge/services/core/api/plugins/data.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/python/fledge/services/core/api/plugins/data.py b/python/fledge/services/core/api/plugins/data.py index ef45129b7..94d4b4658 100644 --- a/python/fledge/services/core/api/plugins/data.py +++ b/python/fledge/services/core/api/plugins/data.py @@ -8,10 +8,12 @@ import urllib.parse from aiohttp import web -from fledge.plugins.common import utils as common_utils from fledge.common.plugin_discovery import PluginDiscovery from fledge.common.storage_client.payload_builder import PayloadBuilder +from fledge.plugins.common import utils as common_utils from fledge.services.core import connect +from fledge.services.core.service_registry.service_registry import ServiceRegistry + __author__ = "Mark Riddoch, Ashish Jabble" __copyright__ = "Copyright (c) 2022 Dianomic Systems Inc." @@ -24,7 +26,7 @@ | GET POST DELETE | /fledge/service/{service_name}/plugin/{plugin_name}/data | --------------------------------------------------------------------------------------- """ - +FORBIDDEN_MSG = "Resource you were trying to reach is absolutely forbidden!" async def get_persist_plugins(request: web.Request) -> web.Response: """ @@ -85,6 +87,10 @@ async def add(request: web.Request) -> web.Response: try: service = request.match_info.get('service_name', None) plugin = request.match_info.get('plugin_name', None) + svc_records = ServiceRegistry.all() + for service_record in svc_records: + if service_record._name == service and int(service_record._status) == 1: + raise web.HTTPForbidden(reason=FORBIDDEN_MSG) storage_client = connect.get_storage_async() await _find_svc_and_plugin(storage_client, service, plugin) key = "{}{}".format(service, plugin) @@ -93,13 +99,13 @@ async def add(request: web.Request) -> web.Response: if response: msg = "{} key already exist.".format(key) return web.HTTPConflict(reason=msg, body=json.dumps({"message": msg})) - data = await request.json() - data = data["data"] + payload = await request.json() + data = payload.get("data") if data is not None: payload = PayloadBuilder().INSERT(key=key, data=data) await storage_client.insert_into_tbl('plugin_data', payload.payload()) else: - raise web.HTTPBadRequest(reason=msg, body=json.dumps({"message": "Malformed data in payload"})) + raise KeyError('Malformed data in payload!') except KeyError as err: msg = str(err) raise web.HTTPBadRequest(reason=msg, body=json.dumps({"message": msg})) From 932841f8ab1306ddee0a79d2e40587941209d330 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Wed, 3 Aug 2022 13:38:06 +0530 Subject: [PATCH 3/3] Forbidden handling added in DELETE plugin data Signed-off-by: ashish-jabble --- python/fledge/services/core/api/plugins/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/fledge/services/core/api/plugins/data.py b/python/fledge/services/core/api/plugins/data.py index 94d4b4658..d6d442507 100644 --- a/python/fledge/services/core/api/plugins/data.py +++ b/python/fledge/services/core/api/plugins/data.py @@ -132,6 +132,10 @@ async def delete(request: web.Request) -> web.Response: service = request.match_info.get('service_name', None) service = urllib.parse.unquote(service) if service is not None else None plugin = request.match_info.get('plugin_name', None) + svc_records = ServiceRegistry.all() + for service_record in svc_records: + if service_record._name == service and int(service_record._status) == 1: + raise web.HTTPForbidden(reason=FORBIDDEN_MSG) storage_client = connect.get_storage_async() key = "{}{}".format(service, plugin) payload = PayloadBuilder().WHERE(['key', '=', key])