diff --git a/src/spaceone/dashboard/manager/data_table_manager/__init__.py b/src/spaceone/dashboard/manager/data_table_manager/__init__.py index 8c6142c..173adc3 100644 --- a/src/spaceone/dashboard/manager/data_table_manager/__init__.py +++ b/src/spaceone/dashboard/manager/data_table_manager/__init__.py @@ -5,15 +5,12 @@ from jinja2 import Environment, meta import pandas as pd -from spaceone.core import cache +from spaceone.core import cache, utils from spaceone.core.manager import BaseManager from spaceone.dashboard.error.data_table import ( - ERROR_REQUIRED_PARAMETER, -) -from spaceone.dashboard.error.data_table import ( - ERROR_QUERY_OPTION, ERROR_NO_FIELDS_TO_GLOBAL_VARIABLES, ERROR_NOT_GLOBAL_VARIABLE_KEY, + ERROR_QUERY_OPTION, ) _LOGGER = logging.getLogger(__name__) @@ -46,21 +43,37 @@ def load( def load_from_widget( self, - query: dict, + granularity: str, + start: str, + end: str, + sort: list = None, + page: dict = None, vars: dict = None, column_sum: bool = False, ) -> dict: - self._check_query(query) - granularity = query["granularity"] - start = query["start"] - end = query["end"] - sort = query.get("sort") - page = query.get("page") - - if cache_data := cache.get( - f"dashboard:Widget:load:{granularity}:{start}:{end}:{vars}:{self.widget_id}:{self.domain_id}" - ): - self.df = pd.DataFrame(cache_data) + + user_id = self.transaction.get_meta( + "authorization.user_id" + ) or self.transaction.get_meta("authorization.app_id") + role_type = self.transaction.get_meta("authorization.role_type") + + query_data = { + "granularity": granularity, + "start": start, + "end": end, + "sort": sort, + "widget_id": self.widget_id, + "domain_id": self.domain_id, + } + + if role_type == "WORKSPACE_MEMBER": + query_data["user_id"] = user_id + + query_hash = utils.dict_to_hash(query_data) + + response = {"results": []} + if cache_data := cache.get(f"dashboard:Widget:load:{query_hash}"): + response = cache_data else: self.load( @@ -70,69 +83,100 @@ def load_from_widget( vars=vars, ) - if column_sum: - return self.response_sum_data() - - return self.response_data(sort, page) - - def make_cache_data(self, granularity, start, end, vars) -> None: - cache_key = f"dashboard:Widget:load:{granularity}:{start}:{end}:{vars}:{self.widget_id}:{self.domain_id}" - if not cache.get(cache_key) and self.df is not None: - cache.set( - cache_key, - self.df.to_dict(orient="records"), - expire=1800, - ) - - @staticmethod - def _check_query(query: dict) -> None: - if "granularity" not in query: - raise ERROR_REQUIRED_PARAMETER(key="query.granularity") + if self.df is not None: + response = { + "results": self.df.copy(deep=True).to_dict(orient="records") + } + cache.set(f"dashboard:Widget:load:{query_hash}", response, expire=600) - if "start" not in query: - raise ERROR_REQUIRED_PARAMETER(key="query.start") + if column_sum: + return self.response_sum_data_from_widget(response) - if "end" not in query: - raise ERROR_REQUIRED_PARAMETER(key="query.end") + return self.response_data_from_widget(response, sort, page) - def response_data(self, sort: list = None, page: dict = None) -> dict: - total_count = len(self.df) + def response_data_from_widget( + self, + response, + sort: list = None, + page: dict = None, + ) -> dict: + data = response["results"] + total_count = len(data) if sort: - self.apply_sort(sort) + data = self.apply_sort(data, sort) if page: - self.apply_page(page) - - df = self.df.copy(deep=True) - self.df = None + data = self.apply_page(data, page) return { - "results": df.to_dict(orient="records"), + "results": data, "total_count": total_count, } - def response_sum_data(self) -> dict: + def response_sum_data_from_widget(self, response) -> dict: + data = response["results"] if self.data_keys: sum_data = { - key: (float(self.df[key].sum())) + key: sum(float(row.get(key, 0)) for row in data) for key in self.data_keys - if key in self.df.columns } else: - numeric_columns = self.df.select_dtypes(include=["float", "int"]).columns - sum_data = {col: float(self.df[col].sum()) for col in numeric_columns} + numeric_columns = { + key + for row in data + for key, value in row.items() + if isinstance(value, (int, float)) + } + sum_data = { + key: sum(float(row.get(key, 0)) for row in data) + for key in numeric_columns + } results = [{column: sum_value} for column, sum_value in sum_data.items()] + return { + "results": results, + "total_count": len(results), + } + + @staticmethod + def apply_sort(data: list, sort: list) -> list: + for rule in reversed(sort): + key = rule["key"] + reverse = rule.get("desc", False) + data = sorted(data, key=lambda item: item[key], reverse=reverse) + return data + + @staticmethod + def apply_page(data: list, page: dict) -> list: + if limit := page.get("limit"): + if limit > 0: + start = page.get("start", 1) + if start < 1: + start = 1 + + start_index = start - 1 + end_index = start_index + limit + return data[start_index:end_index] + def response_data(self, sort: list = None, page: dict = None) -> dict: + total_count = len(self.df) + + if sort: + self.apply_sort_to_df(sort) + + if page: + self.apply_page_df(page) + + df = self.df.copy(deep=True) self.df = None return { - "results": results, - "total_count": len(results), + "results": df.to_dict(orient="records"), + "total_count": total_count, } - def apply_sort(self, sort: list) -> None: + def apply_sort_to_df(self, sort: list) -> None: if len(self.df) > 0: keys = [] ascendings = [] @@ -151,7 +195,7 @@ def apply_sort(self, sort: list) -> None: _LOGGER.error(f"[_sort] Sort Error: {e}") raise ERROR_QUERY_OPTION(key="sort") - def apply_page(self, page: dict) -> None: + def apply_page_df(self, page: dict) -> None: if len(self.df) > 0: if limit := page.get("limit"): if limit > 0: diff --git a/src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py b/src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py index 3361a48..0b46cf9 100644 --- a/src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py +++ b/src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py @@ -112,8 +112,6 @@ def load( self.error_message = e.message if hasattr(e, "message") else str(e) _LOGGER.error(f"[load] add {self.source_type} source error: {e}") - self.make_cache_data(granularity, start, end, vars) - return self.df def _analyze_asset( diff --git a/src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py b/src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py index fccbbe0..e247505 100755 --- a/src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py +++ b/src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py @@ -115,8 +115,6 @@ def load( self.error_message = e.message if hasattr(e, "message") else str(e) _LOGGER.error(f"[load] {self.operator} operation error: {e}") - self.make_cache_data(granularity, start, end, vars) - return self.df def join_data_tables( diff --git a/src/spaceone/dashboard/model/private_widget/request.py b/src/spaceone/dashboard/model/private_widget/request.py index ec1b2b2..fd2f8b1 100644 --- a/src/spaceone/dashboard/model/private_widget/request.py +++ b/src/spaceone/dashboard/model/private_widget/request.py @@ -6,6 +6,7 @@ "PrivateWidgetUpdateRequest", "PrivateWidgetDeleteRequest", "PrivateWidgetLoadRequest", + "PrivateWidgetLoadSumRequest", "PrivateWidgetGetRequest", "PrivateWidgetSearchQueryRequest", ] @@ -48,7 +49,21 @@ class PrivateWidgetDeleteRequest(BaseModel): class PrivateWidgetLoadRequest(BaseModel): widget_id: str - query: dict + granularity: str + start: str + end: str + sort: Union[list, None] = None + page: Union[dict, None] = None + vars: Union[dict, None] = None + user_id: str + domain_id: str + + +class PrivateWidgetLoadSumRequest(BaseModel): + widget_id: str + granularity: str + start: str + end: str vars: Union[dict, None] = None user_id: str domain_id: str diff --git a/src/spaceone/dashboard/model/public_widget/request.py b/src/spaceone/dashboard/model/public_widget/request.py index ef8d410..54b1c78 100644 --- a/src/spaceone/dashboard/model/public_widget/request.py +++ b/src/spaceone/dashboard/model/public_widget/request.py @@ -59,7 +59,11 @@ class PublicWidgetGetRequest(BaseModel): class PublicWidgetLoadRequest(BaseModel): widget_id: str - query: dict + granularity: str + start: str + end: str + sort: Union[list, None] = None + page: Union[dict, None] = None vars: Union[dict, None] = None workspace_id: Union[str, list, None] = None domain_id: str @@ -68,7 +72,9 @@ class PublicWidgetLoadRequest(BaseModel): class PublicWidgetLoadSumRequest(BaseModel): widget_id: str - query: dict + granularity: str + start: str + end: str vars: Union[dict, None] = None workspace_id: Union[str, list, None] = None domain_id: str diff --git a/src/spaceone/dashboard/service/private_widget_service.py b/src/spaceone/dashboard/service/private_widget_service.py index 208cb37..d270253 100644 --- a/src/spaceone/dashboard/service/private_widget_service.py +++ b/src/spaceone/dashboard/service/private_widget_service.py @@ -342,7 +342,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict: Args: params (dict): { 'widget_id': 'str', # required - 'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required + 'granularity': 'str', # required + 'start': 'str', # required + 'end': 'str', # required + 'sort': 'list', + 'page': 'dict', 'vars': 'dict', 'user_id': 'str', # injected from auth (required) 'domain_id': 'str' # injected from auth (required) @@ -379,7 +383,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict: pri_data_table_vo.domain_id, ) return ds_mgr.load_from_widget( - params.query, + params.granularity, + params.start, + params.end, + params.sort, + params.page, params.vars, ) else: @@ -394,7 +402,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict: pri_data_table_vo.domain_id, ) return dt_mgr.load_from_widget( - params.query, + params.granularity, + params.start, + params.end, + params.sort, + params.page, params.vars, ) @@ -405,13 +417,15 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict: @change_value_by_rule("APPEND", "workspace_id", "*") @change_value_by_rule("APPEND", "project_id", "*") @convert_model - def load_sum(self, params: PrivateWidgetLoadRequest) -> dict: + def load_sum(self, params: PrivateWidgetLoadSumRequest) -> dict: """Load private widget Args: params (dict): { 'widget_id': 'str', # required - 'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required + 'granularity': 'str', # required + 'start': 'str', # required + 'end': 'str', # required 'vars': 'dict', 'user_id': 'str', # injected from auth (required) 'domain_id': 'str' # injected from auth (required) @@ -448,8 +462,10 @@ def load_sum(self, params: PrivateWidgetLoadRequest) -> dict: pri_data_table_vo.domain_id, ) return ds_mgr.load_from_widget( - params.query, - params.vars, + params.granularity, + params.start, + params.end, + vars=params.vars, column_sum=True, ) else: @@ -464,8 +480,10 @@ def load_sum(self, params: PrivateWidgetLoadRequest) -> dict: pri_data_table_vo.domain_id, ) return dt_mgr.load_from_widget( - params.query, - params.vars, + params.granularity, + params.start, + params.end, + vars=params.vars, column_sum=True, ) diff --git a/src/spaceone/dashboard/service/public_widget_service.py b/src/spaceone/dashboard/service/public_widget_service.py index d635431..a53893d 100644 --- a/src/spaceone/dashboard/service/public_widget_service.py +++ b/src/spaceone/dashboard/service/public_widget_service.py @@ -353,7 +353,11 @@ def load(self, params: PublicWidgetLoadRequest) -> dict: Args: params (dict): { 'widget_id': 'str', # required - 'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required + 'granularity': 'str', # required + 'start': 'str', # required + 'end': 'str', # required + 'sort': 'list', + 'page': 'dict', 'vars': 'dict', 'workspace_id': 'str', # injected from auth 'domain_id': 'str' # injected from auth (required) @@ -393,7 +397,11 @@ def load(self, params: PublicWidgetLoadRequest) -> dict: pub_data_table_vo.domain_id, ) return ds_mgr.load_from_widget( - params.query, + params.granularity, + params.start, + params.end, + params.sort, + params.page, params.vars, ) else: @@ -408,7 +416,11 @@ def load(self, params: PublicWidgetLoadRequest) -> dict: pub_data_table_vo.domain_id, ) return dt_mgr.load_from_widget( - params.query, + params.granularity, + params.start, + params.end, + params.sort, + params.page, params.vars, ) @@ -425,7 +437,9 @@ def load_sum(self, params: PublicWidgetLoadSumRequest) -> dict: Args: params (dict): { 'widget_id': 'str', # required - 'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required + 'granularity': 'str', # required + 'start': 'str', # required + 'end': 'str', # required 'vars': 'dict', 'workspace_id': 'str', # injected from auth 'domain_id': 'str' # injected from auth (required) @@ -465,8 +479,10 @@ def load_sum(self, params: PublicWidgetLoadSumRequest) -> dict: pub_data_table_vo.domain_id, ) return ds_mgr.load_from_widget( - params.query, - params.vars, + params.granularity, + params.start, + params.end, + vars=params.vars, column_sum=True, ) else: @@ -481,8 +497,10 @@ def load_sum(self, params: PublicWidgetLoadSumRequest) -> dict: pub_data_table_vo.domain_id, ) return dt_mgr.load_from_widget( - params.query, - params.vars, + params.granularity, + params.start, + params.end, + vars=params.vars, column_sum=True, )