Merge pull request #123 from stat-kwon/master

Apply it in a way that does not use DataFrame for cache application

src/spaceone/dashboard/manager/data_table_manager/__init__.py

@@ -5,15 +5,12 @@
 from jinja2 import Environment, meta
 import pandas as pd
 
-from spaceone.core import cache
+from spaceone.core import cache, utils
 from spaceone.core.manager import BaseManager
 from spaceone.dashboard.error.data_table import (
-    ERROR_REQUIRED_PARAMETER,
-)
-from spaceone.dashboard.error.data_table import (
-    ERROR_QUERY_OPTION,
     ERROR_NO_FIELDS_TO_GLOBAL_VARIABLES,
     ERROR_NOT_GLOBAL_VARIABLE_KEY,
+    ERROR_QUERY_OPTION,
 )
 
 _LOGGER = logging.getLogger(__name__)
@@ -46,21 +43,37 @@ def load(
 
     def load_from_widget(
         self,
-        query: dict,
+        granularity: str,
+        start: str,
+        end: str,
+        sort: list = None,
+        page: dict = None,
         vars: dict = None,
         column_sum: bool = False,
     ) -> dict:
-        self._check_query(query)
-        granularity = query["granularity"]
-        start = query["start"]
-        end = query["end"]
-        sort = query.get("sort")
-        page = query.get("page")
-
-        if cache_data := cache.get(
-            f"dashboard:Widget:load:{granularity}:{start}:{end}:{vars}:{self.widget_id}:{self.domain_id}"
-        ):
-            self.df = pd.DataFrame(cache_data)
+
+        user_id = self.transaction.get_meta(
+            "authorization.user_id"
+        ) or self.transaction.get_meta("authorization.app_id")
+        role_type = self.transaction.get_meta("authorization.role_type")
+
+        query_data = {
+            "granularity": granularity,
+            "start": start,
+            "end": end,
+            "sort": sort,
+            "widget_id": self.widget_id,
+            "domain_id": self.domain_id,
+        }
+
+        if role_type == "WORKSPACE_MEMBER":
+            query_data["user_id"] = user_id
+
+        query_hash = utils.dict_to_hash(query_data)
+
+        response = {"results": []}
+        if cache_data := cache.get(f"dashboard:Widget:load:{query_hash}"):
+            response = cache_data
 
         else:
             self.load(
@@ -70,69 +83,100 @@ def load_from_widget(
                 vars=vars,
             )
 
-        if column_sum:
-            return self.response_sum_data()
-
-        return self.response_data(sort, page)
-
-    def make_cache_data(self, granularity, start, end, vars) -> None:
-        cache_key = f"dashboard:Widget:load:{granularity}:{start}:{end}:{vars}:{self.widget_id}:{self.domain_id}"
-        if not cache.get(cache_key) and self.df is not None:
-            cache.set(
-                cache_key,
-                self.df.to_dict(orient="records"),
-                expire=1800,
-            )
-
-    @staticmethod
-    def _check_query(query: dict) -> None:
-        if "granularity" not in query:
-            raise ERROR_REQUIRED_PARAMETER(key="query.granularity")
+            if self.df is not None:
+                response = {
+                    "results": self.df.copy(deep=True).to_dict(orient="records")
+                }
+                cache.set(f"dashboard:Widget:load:{query_hash}", response, expire=600)
 
-        if "start" not in query:
-            raise ERROR_REQUIRED_PARAMETER(key="query.start")
+        if column_sum:
+            return self.response_sum_data_from_widget(response)
 
-        if "end" not in query:
-            raise ERROR_REQUIRED_PARAMETER(key="query.end")
+        return self.response_data_from_widget(response, sort, page)
 
-    def response_data(self, sort: list = None, page: dict = None) -> dict:
-        total_count = len(self.df)
+    def response_data_from_widget(
+        self,
+        response,
+        sort: list = None,
+        page: dict = None,
+    ) -> dict:
+        data = response["results"]
+        total_count = len(data)
 
         if sort:
-            self.apply_sort(sort)
+            data = self.apply_sort(data, sort)
 
         if page:
-            self.apply_page(page)
-
-        df = self.df.copy(deep=True)
-        self.df = None
+            data = self.apply_page(data, page)
 
         return {
-            "results": df.to_dict(orient="records"),
+            "results": data,
             "total_count": total_count,
         }
 
-    def response_sum_data(self) -> dict:
+    def response_sum_data_from_widget(self, response) -> dict:
+        data = response["results"]
         if self.data_keys:
             sum_data = {
-                key: (float(self.df[key].sum()))
+                key: sum(float(row.get(key, 0)) for row in data)
                 for key in self.data_keys
-                if key in self.df.columns
             }
         else:
-            numeric_columns = self.df.select_dtypes(include=["float", "int"]).columns
-            sum_data = {col: float(self.df[col].sum()) for col in numeric_columns}
+            numeric_columns = {
+                key
+                for row in data
+                for key, value in row.items()
+                if isinstance(value, (int, float))
+            }
+            sum_data = {
+                key: sum(float(row.get(key, 0)) for row in data)
+                for key in numeric_columns
+            }
 
         results = [{column: sum_value} for column, sum_value in sum_data.items()]
+        return {
+            "results": results,
+            "total_count": len(results),
+        }
+
+    @staticmethod
+    def apply_sort(data: list, sort: list) -> list:
+        for rule in reversed(sort):
+            key = rule["key"]
+            reverse = rule.get("desc", False)
+            data = sorted(data, key=lambda item: item[key], reverse=reverse)
+        return data
+
+    @staticmethod
+    def apply_page(data: list, page: dict) -> list:
+        if limit := page.get("limit"):
+            if limit > 0:
+                start = page.get("start", 1)
+                if start < 1:
+                    start = 1
+
+                start_index = start - 1
+                end_index = start_index + limit
+                return data[start_index:end_index]
 
+    def response_data(self, sort: list = None, page: dict = None) -> dict:
+        total_count = len(self.df)
+
+        if sort:
+            self.apply_sort_to_df(sort)
+
+        if page:
+            self.apply_page_df(page)
+
+        df = self.df.copy(deep=True)
         self.df = None
 
         return {
-            "results": results,
-            "total_count": len(results),
+            "results": df.to_dict(orient="records"),
+            "total_count": total_count,
         }
 
-    def apply_sort(self, sort: list) -> None:
+    def apply_sort_to_df(self, sort: list) -> None:
         if len(self.df) > 0:
             keys = []
             ascendings = []
@@ -151,7 +195,7 @@ def apply_sort(self, sort: list) -> None:
                 _LOGGER.error(f"[_sort] Sort Error: {e}")
                 raise ERROR_QUERY_OPTION(key="sort")
 
-    def apply_page(self, page: dict) -> None:
+    def apply_page_df(self, page: dict) -> None:
         if len(self.df) > 0:
             if limit := page.get("limit"):
                 if limit > 0:

src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py

@@ -112,8 +112,6 @@ def load(
             self.error_message = e.message if hasattr(e, "message") else str(e)
             _LOGGER.error(f"[load] add {self.source_type} source error: {e}")
 
-        self.make_cache_data(granularity, start, end, vars)
-
         return self.df
 
     def _analyze_asset(

src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py

@@ -115,8 +115,6 @@ def load(
             self.error_message = e.message if hasattr(e, "message") else str(e)
             _LOGGER.error(f"[load] {self.operator} operation error: {e}")
 
-        self.make_cache_data(granularity, start, end, vars)
-
         return self.df
 
     def join_data_tables(

src/spaceone/dashboard/model/private_widget/request.py

@@ -6,6 +6,7 @@
     "PrivateWidgetUpdateRequest",
     "PrivateWidgetDeleteRequest",
     "PrivateWidgetLoadRequest",
+    "PrivateWidgetLoadSumRequest",
     "PrivateWidgetGetRequest",
     "PrivateWidgetSearchQueryRequest",
 ]
@@ -48,7 +49,21 @@ class PrivateWidgetDeleteRequest(BaseModel):
 
 class PrivateWidgetLoadRequest(BaseModel):
     widget_id: str
-    query: dict
+    granularity: str
+    start: str
+    end: str
+    sort: Union[list, None] = None
+    page: Union[dict, None] = None
+    vars: Union[dict, None] = None
+    user_id: str
+    domain_id: str
+
+
+class PrivateWidgetLoadSumRequest(BaseModel):
+    widget_id: str
+    granularity: str
+    start: str
+    end: str
     vars: Union[dict, None] = None
     user_id: str
     domain_id: str

src/spaceone/dashboard/model/public_widget/request.py

@@ -59,7 +59,11 @@ class PublicWidgetGetRequest(BaseModel):
 
 class PublicWidgetLoadRequest(BaseModel):
     widget_id: str
-    query: dict
+    granularity: str
+    start: str
+    end: str
+    sort: Union[list, None] = None
+    page: Union[dict, None] = None
     vars: Union[dict, None] = None
     workspace_id: Union[str, list, None] = None
     domain_id: str
@@ -68,7 +72,9 @@ class PublicWidgetLoadRequest(BaseModel):
 
 class PublicWidgetLoadSumRequest(BaseModel):
     widget_id: str
-    query: dict
+    granularity: str
+    start: str
+    end: str
     vars: Union[dict, None] = None
     workspace_id: Union[str, list, None] = None
     domain_id: str

src/spaceone/dashboard/service/private_widget_service.py

@@ -342,7 +342,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict:
         Args:
             params (dict): {
                 'widget_id': 'str',             # required
-                'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required
+                'granularity': 'str',           # required
+                'start': 'str',                 # required
+                'end': 'str',                   # required
+                'sort': 'list',
+                'page': 'dict',
                 'vars': 'dict',
                 'user_id': 'str',               # injected from auth (required)
                 'domain_id': 'str'              # injected from auth (required)
@@ -379,7 +383,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict:
                 pri_data_table_vo.domain_id,
             )
             return ds_mgr.load_from_widget(
-                params.query,
+                params.granularity,
+                params.start,
+                params.end,
+                params.sort,
+                params.page,
                 params.vars,
             )
         else:
@@ -394,7 +402,11 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict:
                 pri_data_table_vo.domain_id,
             )
             return dt_mgr.load_from_widget(
-                params.query,
+                params.granularity,
+                params.start,
+                params.end,
+                params.sort,
+                params.page,
                 params.vars,
             )
 
@@ -405,13 +417,15 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict:
     @change_value_by_rule("APPEND", "workspace_id", "*")
     @change_value_by_rule("APPEND", "project_id", "*")
     @convert_model
-    def load_sum(self, params: PrivateWidgetLoadRequest) -> dict:
+    def load_sum(self, params: PrivateWidgetLoadSumRequest) -> dict:
         """Load private widget
 
         Args:
             params (dict): {
                 'widget_id': 'str',             # required
-                'query': 'dict (spaceone.api.core.v1.AnalyzeQuery)', # required
+                'granularity': 'str',           # required
+                'start': 'str',                 # required
+                'end': 'str',                   # required
                 'vars': 'dict',
                 'user_id': 'str',               # injected from auth (required)
                 'domain_id': 'str'              # injected from auth (required)
@@ -448,8 +462,10 @@ def load_sum(self, params: PrivateWidgetLoadRequest) -> dict:
                 pri_data_table_vo.domain_id,
             )
             return ds_mgr.load_from_widget(
-                params.query,
-                params.vars,
+                params.granularity,
+                params.start,
+                params.end,
+                vars=params.vars,
                 column_sum=True,
             )
         else:
@@ -464,8 +480,10 @@ def load_sum(self, params: PrivateWidgetLoadRequest) -> dict:
                 pri_data_table_vo.domain_id,
             )
             return dt_mgr.load_from_widget(
-                params.query,
-                params.vars,
+                params.granularity,
+                params.start,
+                params.end,
+                vars=params.vars,
                 column_sum=True,
             )