From 2d57d3897c4e6019c2a54f4ea0230d8123e611dd Mon Sep 17 00:00:00 2001 From: seolmin Date: Mon, 16 Dec 2024 23:35:39 +0900 Subject: [PATCH 1/3] fix: apply it in a way that does not use DataFrame for cache application Signed-off-by: seolmin --- .../manager/data_table_manager/__init__.py | 120 ++++++++++-------- .../data_table_manager/data_source_manager.py | 2 - .../data_transformation_manager.py | 2 - 3 files changed, 69 insertions(+), 55 deletions(-) diff --git a/src/spaceone/dashboard/manager/data_table_manager/__init__.py b/src/spaceone/dashboard/manager/data_table_manager/__init__.py index 8c6142c..ec1bde8 100644 --- a/src/spaceone/dashboard/manager/data_table_manager/__init__.py +++ b/src/spaceone/dashboard/manager/data_table_manager/__init__.py @@ -5,7 +5,7 @@ from jinja2 import Environment, meta import pandas as pd -from spaceone.core import cache +from spaceone.core import cache, utils from spaceone.core.manager import BaseManager from spaceone.dashboard.error.data_table import ( ERROR_REQUIRED_PARAMETER, @@ -57,10 +57,28 @@ def load_from_widget( sort = query.get("sort") page = query.get("page") - if cache_data := cache.get( - f"dashboard:Widget:load:{granularity}:{start}:{end}:{vars}:{self.widget_id}:{self.domain_id}" - ): - self.df = pd.DataFrame(cache_data) + user_id = self.transaction.get_meta( + "authorization.user_id" + ) or self.transaction.get_meta("authorization.app_id") + role_type = self.transaction.get_meta("authorization.role_type") + + query_data = { + "granularity": granularity, + "start": start, + "end": end, + "sort": sort, + "widget_id": self.widget_id, + "domain_id": self.domain_id, + } + + if role_type == "WORKSPACE_MEMBER": + query_data["user_id"] = user_id + + query_hash = utils.dict_to_hash(query_data) + + response = {"results": []} + if cache_data := cache.get(f"dashboard:Widget:load:{query_hash}"): + response = cache_data else: self.load( @@ -70,10 +88,16 @@ def load_from_widget( vars=vars, ) + if self.df is not None: + response = { + "results": self.df.copy(deep=True).to_dict(orient="records") + } + cache.set(f"dashboard:Widget:load:{query_hash}", response, expire=600) + if column_sum: - return self.response_sum_data() + return self.response_sum_data(response) - return self.response_data(sort, page) + return self.response_data(response, sort, page) def make_cache_data(self, granularity, start, end, vars) -> None: cache_key = f"dashboard:Widget:load:{granularity}:{start}:{end}:{vars}:{self.widget_id}:{self.domain_id}" @@ -95,71 +119,65 @@ def _check_query(query: dict) -> None: if "end" not in query: raise ERROR_REQUIRED_PARAMETER(key="query.end") - def response_data(self, sort: list = None, page: dict = None) -> dict: - total_count = len(self.df) + def response_data(self, response, sort: list = None, page: dict = None) -> dict: + data = response["results"] + total_count = len(data) if sort: - self.apply_sort(sort) + data = self.apply_sort(data, sort) if page: - self.apply_page(page) - - df = self.df.copy(deep=True) - self.df = None + data = self.apply_page(data, page) return { - "results": df.to_dict(orient="records"), + "results": data, "total_count": total_count, } - def response_sum_data(self) -> dict: + def response_sum_data(self, response) -> dict: + data = response["results"] if self.data_keys: sum_data = { - key: (float(self.df[key].sum())) + key: sum(float(row.get(key, 0)) for row in data) for key in self.data_keys - if key in self.df.columns } else: - numeric_columns = self.df.select_dtypes(include=["float", "int"]).columns - sum_data = {col: float(self.df[col].sum()) for col in numeric_columns} + numeric_columns = { + key + for row in data + for key, value in row.items() + if isinstance(value, (int, float)) + } + sum_data = { + key: sum(float(row.get(key, 0)) for row in data) + for key in numeric_columns + } results = [{column: sum_value} for column, sum_value in sum_data.items()] - - self.df = None - return { "results": results, "total_count": len(results), } - def apply_sort(self, sort: list) -> None: - if len(self.df) > 0: - keys = [] - ascendings = [] - - for sort_option in sort: - key = sort_option.get("key") - ascending = not sort_option.get("desc", False) - - if key: - keys.append(key) - ascendings.append(ascending) - - try: - self.df = self.df.sort_values(by=keys, ascending=ascendings) - except Exception as e: - _LOGGER.error(f"[_sort] Sort Error: {e}") - raise ERROR_QUERY_OPTION(key="sort") - - def apply_page(self, page: dict) -> None: - if len(self.df) > 0: - if limit := page.get("limit"): - if limit > 0: - start = page.get("start", 1) - if start < 1: - start = 1 - - self.df = self.df.iloc[start - 1 : start + limit - 1] + @staticmethod + def apply_sort(data: list, sort: list) -> list: + for rule in reversed(sort): + key = rule["key"] + reverse = rule.get("desc", False) + data = sorted(data, key=lambda item: item[key], reverse=reverse) + return data + + @staticmethod + def apply_page(data: list, page: dict) -> list: + if limit := page.get("limit"): + if limit > 0: + start = page.get("start", 1) + if start < 1: + start = 1 + + start_index = start - 1 + end_index = start_index + limit + return data[start_index:end_index] def is_jinja_expression(self, expression: str) -> bool: env = Environment() diff --git a/src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py b/src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py index 3361a48..0b46cf9 100644 --- a/src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py +++ b/src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py @@ -112,8 +112,6 @@ def load( self.error_message = e.message if hasattr(e, "message") else str(e) _LOGGER.error(f"[load] add {self.source_type} source error: {e}") - self.make_cache_data(granularity, start, end, vars) - return self.df def _analyze_asset( diff --git a/src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py b/src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py index fccbbe0..e247505 100755 --- a/src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py +++ b/src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py @@ -115,8 +115,6 @@ def load( self.error_message = e.message if hasattr(e, "message") else str(e) _LOGGER.error(f"[load] {self.operator} operation error: {e}") - self.make_cache_data(granularity, start, end, vars) - return self.df def join_data_tables( From adb2cc799d0e629856c3619cdcc4e3b663f2fa55 Mon Sep 17 00:00:00 2001 From: seolmin Date: Tue, 17 Dec 2024 00:27:27 +0900 Subject: [PATCH 2/3] fix: change interface of load and load_sum methods Signed-off-by: seolmin --- .../manager/data_table_manager/__init__.py | 36 +++---------------- .../dashboard/model/private_widget/request.py | 17 ++++++++- .../dashboard/model/public_widget/request.py | 10 ++++-- .../service/private_widget_service.py | 36 ++++++++++++++----- .../service/public_widget_service.py | 34 +++++++++++++----- 5 files changed, 82 insertions(+), 51 deletions(-) diff --git a/src/spaceone/dashboard/manager/data_table_manager/__init__.py b/src/spaceone/dashboard/manager/data_table_manager/__init__.py index ec1bde8..393b3be 100644 --- a/src/spaceone/dashboard/manager/data_table_manager/__init__.py +++ b/src/spaceone/dashboard/manager/data_table_manager/__init__.py @@ -8,10 +8,6 @@ from spaceone.core import cache, utils from spaceone.core.manager import BaseManager from spaceone.dashboard.error.data_table import ( - ERROR_REQUIRED_PARAMETER, -) -from spaceone.dashboard.error.data_table import ( - ERROR_QUERY_OPTION, ERROR_NO_FIELDS_TO_GLOBAL_VARIABLES, ERROR_NOT_GLOBAL_VARIABLE_KEY, ) @@ -46,16 +42,14 @@ def load( def load_from_widget( self, - query: dict, + granularity: str, + start: str, + end: str, + sort: list = None, + page: dict = None, vars: dict = None, column_sum: bool = False, ) -> dict: - self._check_query(query) - granularity = query["granularity"] - start = query["start"] - end = query["end"] - sort = query.get("sort") - page = query.get("page") user_id = self.transaction.get_meta( "authorization.user_id" @@ -99,26 +93,6 @@ def load_from_widget( return self.response_data(response, sort, page) - def make_cache_data(self, granularity, start, end, vars) -> None: - cache_key = f"dashboard:Widget:load:{granularity}:{start}:{end}:{vars}:{self.widget_id}:{self.domain_id}" - if not cache.get(cache_key) and self.df is not None: - cache.set( - cache_key, - self.df.to_dict(orient="records"), - expire=1800, - ) - - @staticmethod - def _check_query(query: dict) -> None: - if "granularity" not in query: - raise ERROR_REQUIRED_PARAMETER(key="query.granularity") - - if "start" not in query: - raise ERROR_REQUIRED_PARAMETER(key="query.start") - - if "end" not in query: - raise ERROR_REQUIRED_PARAMETER(key="query.end") - def response_data(self, response, sort: list = None, page: dict = None) -> dict: data = response["results"] total_count = len(data) diff --git a/src/spaceone/dashboard/model/private_widget/request.py b/src/spaceone/dashboard/model/private_widget/request.py index ec1b2b2..fd2f8b1 100644 --- a/src/spaceone/dashboard/model/private_widget/request.py +++ b/src/spaceone/dashboard/model/private_widget/request.py @@ -6,6 +6,7 @@ "PrivateWidgetUpdateRequest", "PrivateWidgetDeleteRequest", "PrivateWidgetLoadRequest", + "PrivateWidgetLoadSumRequest", "PrivateWidgetGetRequest", "PrivateWidgetSearchQueryRequest", ] @@ -48,7 +49,21 @@ class PrivateWidgetDeleteRequest(BaseModel): class PrivateWidgetLoadRequest(BaseModel): widget_id: str - query: dict + granularity: str + start: str + end: str + sort: Union[list, None] = None + page: Union[dict, None] = None + vars: Union[dict, None] = None + user_id: str + domain_id: str + + +class PrivateWidgetLoadSumRequest(BaseModel): + widget_id: str + granularity: str + start: str + end: str vars: Union[dict, None] = None user_id: str domain_id: str diff --git a/src/spaceone/dashboard/model/public_widget/request.py b/src/spaceone/dashboard/model/public_widget/request.py index ef8d410..54b1c78 100644 --- a/src/spaceone/dashboard/model/public_widget/request.py +++ b/src/spaceone/dashboard/model/public_widget/request.py @@ -59,7 +59,11 @@ class PublicWidgetGetRequest(BaseModel): class PublicWidgetLoadRequest(BaseModel): widget_id: str - query: dict + granularity: str + start: str + end: str + sort: Union[list, None] = None + page: Union[dict, None] = None vars: Union[dict, None] = None workspace_id: Union[str, list, None] = None domain_id: str @@ -68,7 +72,9 @@ class PublicWidgetLoadRequest(BaseModel): class PublicWidgetLoadSumRequest(BaseModel): widget_id: str - query: dict + granularity: str + start: str + end: str vars: Union[dict, None] = None workspace_id: Union[str, list, None] = None domain_id: str diff --git a/src/spaceone/dashboard/service/private_widget_service.py b/src/spaceone/dashboard/service/private_widget_service.py index 208cb37..d270253 100644 --- a/src/spaceone/dashboard/service/private_widget_service.py +++ b/src/spaceone/dashboard/service/private_widget_service.py @@ -342,7 +342,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict: Args: params (dict): { 'widget_id': 'str', # required - 'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required + 'granularity': 'str', # required + 'start': 'str', # required + 'end': 'str', # required + 'sort': 'list', + 'page': 'dict', 'vars': 'dict', 'user_id': 'str', # injected from auth (required) 'domain_id': 'str' # injected from auth (required) @@ -379,7 +383,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict: pri_data_table_vo.domain_id, ) return ds_mgr.load_from_widget( - params.query, + params.granularity, + params.start, + params.end, + params.sort, + params.page, params.vars, ) else: @@ -394,7 +402,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict: pri_data_table_vo.domain_id, ) return dt_mgr.load_from_widget( - params.query, + params.granularity, + params.start, + params.end, + params.sort, + params.page, params.vars, ) @@ -405,13 +417,15 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict: @change_value_by_rule("APPEND", "workspace_id", "*") @change_value_by_rule("APPEND", "project_id", "*") @convert_model - def load_sum(self, params: PrivateWidgetLoadRequest) -> dict: + def load_sum(self, params: PrivateWidgetLoadSumRequest) -> dict: """Load private widget Args: params (dict): { 'widget_id': 'str', # required - 'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required + 'granularity': 'str', # required + 'start': 'str', # required + 'end': 'str', # required 'vars': 'dict', 'user_id': 'str', # injected from auth (required) 'domain_id': 'str' # injected from auth (required) @@ -448,8 +462,10 @@ def load_sum(self, params: PrivateWidgetLoadRequest) -> dict: pri_data_table_vo.domain_id, ) return ds_mgr.load_from_widget( - params.query, - params.vars, + params.granularity, + params.start, + params.end, + vars=params.vars, column_sum=True, ) else: @@ -464,8 +480,10 @@ def load_sum(self, params: PrivateWidgetLoadRequest) -> dict: pri_data_table_vo.domain_id, ) return dt_mgr.load_from_widget( - params.query, - params.vars, + params.granularity, + params.start, + params.end, + vars=params.vars, column_sum=True, ) diff --git a/src/spaceone/dashboard/service/public_widget_service.py b/src/spaceone/dashboard/service/public_widget_service.py index d635431..a53893d 100644 --- a/src/spaceone/dashboard/service/public_widget_service.py +++ b/src/spaceone/dashboard/service/public_widget_service.py @@ -353,7 +353,11 @@ def load(self, params: PublicWidgetLoadRequest) -> dict: Args: params (dict): { 'widget_id': 'str', # required - 'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required + 'granularity': 'str', # required + 'start': 'str', # required + 'end': 'str', # required + 'sort': 'list', + 'page': 'dict', 'vars': 'dict', 'workspace_id': 'str', # injected from auth 'domain_id': 'str' # injected from auth (required) @@ -393,7 +397,11 @@ def load(self, params: PublicWidgetLoadRequest) -> dict: pub_data_table_vo.domain_id, ) return ds_mgr.load_from_widget( - params.query, + params.granularity, + params.start, + params.end, + params.sort, + params.page, params.vars, ) else: @@ -408,7 +416,11 @@ def load(self, params: PublicWidgetLoadRequest) -> dict: pub_data_table_vo.domain_id, ) return dt_mgr.load_from_widget( - params.query, + params.granularity, + params.start, + params.end, + params.sort, + params.page, params.vars, ) @@ -425,7 +437,9 @@ def load_sum(self, params: PublicWidgetLoadSumRequest) -> dict: Args: params (dict): { 'widget_id': 'str', # required - 'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required + 'granularity': 'str', # required + 'start': 'str', # required + 'end': 'str', # required 'vars': 'dict', 'workspace_id': 'str', # injected from auth 'domain_id': 'str' # injected from auth (required) @@ -465,8 +479,10 @@ def load_sum(self, params: PublicWidgetLoadSumRequest) -> dict: pub_data_table_vo.domain_id, ) return ds_mgr.load_from_widget( - params.query, - params.vars, + params.granularity, + params.start, + params.end, + vars=params.vars, column_sum=True, ) else: @@ -481,8 +497,10 @@ def load_sum(self, params: PublicWidgetLoadSumRequest) -> dict: pub_data_table_vo.domain_id, ) return dt_mgr.load_from_widget( - params.query, - params.vars, + params.granularity, + params.start, + params.end, + vars=params.vars, column_sum=True, ) From 35bbdaf3bb8f40d5462885b6fc96db27944bac87 Mon Sep 17 00:00:00 2001 From: seolmin Date: Tue, 17 Dec 2024 00:55:44 +0900 Subject: [PATCH 3/3] fix: Change the load response logic of DataTable and Widget Signed-off-by: seolmin --- .../manager/data_table_manager/__init__.py | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/src/spaceone/dashboard/manager/data_table_manager/__init__.py b/src/spaceone/dashboard/manager/data_table_manager/__init__.py index 393b3be..173adc3 100644 --- a/src/spaceone/dashboard/manager/data_table_manager/__init__.py +++ b/src/spaceone/dashboard/manager/data_table_manager/__init__.py @@ -10,6 +10,7 @@ from spaceone.dashboard.error.data_table import ( ERROR_NO_FIELDS_TO_GLOBAL_VARIABLES, ERROR_NOT_GLOBAL_VARIABLE_KEY, + ERROR_QUERY_OPTION, ) _LOGGER = logging.getLogger(__name__) @@ -89,11 +90,16 @@ def load_from_widget( cache.set(f"dashboard:Widget:load:{query_hash}", response, expire=600) if column_sum: - return self.response_sum_data(response) + return self.response_sum_data_from_widget(response) - return self.response_data(response, sort, page) + return self.response_data_from_widget(response, sort, page) - def response_data(self, response, sort: list = None, page: dict = None) -> dict: + def response_data_from_widget( + self, + response, + sort: list = None, + page: dict = None, + ) -> dict: data = response["results"] total_count = len(data) @@ -108,7 +114,7 @@ def response_data(self, response, sort: list = None, page: dict = None) -> dict: "total_count": total_count, } - def response_sum_data(self, response) -> dict: + def response_sum_data_from_widget(self, response) -> dict: data = response["results"] if self.data_keys: sum_data = { @@ -153,6 +159,52 @@ def apply_page(data: list, page: dict) -> list: end_index = start_index + limit return data[start_index:end_index] + def response_data(self, sort: list = None, page: dict = None) -> dict: + total_count = len(self.df) + + if sort: + self.apply_sort_to_df(sort) + + if page: + self.apply_page_df(page) + + df = self.df.copy(deep=True) + self.df = None + + return { + "results": df.to_dict(orient="records"), + "total_count": total_count, + } + + def apply_sort_to_df(self, sort: list) -> None: + if len(self.df) > 0: + keys = [] + ascendings = [] + + for sort_option in sort: + key = sort_option.get("key") + ascending = not sort_option.get("desc", False) + + if key: + keys.append(key) + ascendings.append(ascending) + + try: + self.df = self.df.sort_values(by=keys, ascending=ascendings) + except Exception as e: + _LOGGER.error(f"[_sort] Sort Error: {e}") + raise ERROR_QUERY_OPTION(key="sort") + + def apply_page_df(self, page: dict) -> None: + if len(self.df) > 0: + if limit := page.get("limit"): + if limit > 0: + start = page.get("start", 1) + if start < 1: + start = 1 + + self.df = self.df.iloc[start - 1 : start + limit - 1] + def is_jinja_expression(self, expression: str) -> bool: env = Environment()