From bea918e664522533dfe062c0a8761747509d8db1 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Fri, 26 Jan 2024 07:34:08 +0900 Subject: [PATCH] feat: add cost_report_data analyze api --- .../interface/grpc/cost_report_data.py | 6 + .../task/v1/data_source_sync_scheduler.py | 30 +++ .../manager/cost_report_config_manager.py | 14 +- .../cost_analysis/manager/email_manager.py | 4 +- .../cost_analysis/manager/identity_manager.py | 10 +- src/spaceone/cost_analysis/model/__init__.py | 3 + .../model/cost_report/database.py | 5 +- .../model/cost_report_data/database.py | 5 +- .../service/cost_report_config_service.py | 8 +- .../service/cost_report_data_service.py | 53 ++--- .../service/cost_report_serivce.py | 182 +++++++++++++----- 11 files changed, 220 insertions(+), 100 deletions(-) diff --git a/src/spaceone/cost_analysis/interface/grpc/cost_report_data.py b/src/spaceone/cost_analysis/interface/grpc/cost_report_data.py index c8742f7a..ec631ca8 100644 --- a/src/spaceone/cost_analysis/interface/grpc/cost_report_data.py +++ b/src/spaceone/cost_analysis/interface/grpc/cost_report_data.py @@ -24,3 +24,9 @@ def analyze(self, request, context): cost_report_data_svc = CostReportDataService(metadata) response: dict = cost_report_data_svc.analyze(params) return self.dict_to_message(response) + + def stat(self, request, context): + params, metadata = self.parse_request(request, context) + cost_report_data_svc = CostReportDataService(metadata) + response: dict = cost_report_data_svc.stat(params) + return self.dict_to_message(response) diff --git a/src/spaceone/cost_analysis/interface/task/v1/data_source_sync_scheduler.py b/src/spaceone/cost_analysis/interface/task/v1/data_source_sync_scheduler.py index 85416cb7..02a0de77 100644 --- a/src/spaceone/cost_analysis/interface/task/v1/data_source_sync_scheduler.py +++ b/src/spaceone/cost_analysis/interface/task/v1/data_source_sync_scheduler.py @@ -24,6 +24,7 @@ def _init_config(self): def create_task(self) -> list: tasks = [] + tasks.extend(self._send_cost_report_task()) tasks.extend(self._create_cost_report_run_task()) tasks.extend(self._create_data_source_sync_task()) return tasks @@ -99,3 +100,32 @@ def _create_cost_report_run_task(self): f"{utils.datetime_to_iso8601(datetime.utcnow())} [INFO] [create_task] data_source_sync_time: {self._data_source_sync_hour} hour (UTC)" ) return [] + + def _send_cost_report_task(self): + if datetime.utcnow().hour == self._data_source_sync_hour: + stp = { + "name": "cost_report_run_schedule", + "version": "v1", + "executionEngine": "BaseWorker", + "stages": [ + { + "locator": "SERVICE", + "name": "CostReportService", + "metadata": {"token": self._token}, + "method": "send_cost_report_by_cost_report_config", + "params": {"params": {}}, + } + ], + } + print( + f"{utils.datetime_to_iso8601(datetime.utcnow())} [INFO] [create_task] send_cost_report_by_cost_report_config => START" + ) + return [stp] + else: + print( + f"{utils.datetime_to_iso8601(datetime.utcnow())} [INFO] [create_task] send_cost_report_by_cost_report_config => SKIP" + ) + print( + f"{utils.datetime_to_iso8601(datetime.utcnow())} [INFO] [create_task] data_source_sync_time: {self._data_source_sync_hour} hour (UTC)" + ) + return [] diff --git a/src/spaceone/cost_analysis/manager/cost_report_config_manager.py b/src/spaceone/cost_analysis/manager/cost_report_config_manager.py index 8d5bfd40..460a914d 100644 --- a/src/spaceone/cost_analysis/manager/cost_report_config_manager.py +++ b/src/spaceone/cost_analysis/manager/cost_report_config_manager.py @@ -81,17 +81,11 @@ def run_cost_report_config(self, cost_report_config_vo: CostReportConfig) -> Non pass def get_cost_report_config( - self, cost_report_config_id: str, domain_id: str, workspace_id: str = None + self, cost_report_config_id: str, domain_id: str ) -> CostReportConfig: - conditions = { - "cost_report_config_id": cost_report_config_id, - "domain_id": domain_id, - } - - if workspace_id: - conditions["workspace_id"] = workspace_id - - return self.cost_report_config_model.get(**conditions) + return self.cost_report_config_model.get( + domain_id=domain_id, cost_report_config_id=cost_report_config_id + ) def list_cost_report_config(self, query: dict, domain_id) -> Tuple[QuerySet, int]: self._create_default_cost_report_config(domain_id) diff --git a/src/spaceone/cost_analysis/manager/email_manager.py b/src/spaceone/cost_analysis/manager/email_manager.py index e661673e..ff8ad9af 100644 --- a/src/spaceone/cost_analysis/manager/email_manager.py +++ b/src/spaceone/cost_analysis/manager/email_manager.py @@ -48,7 +48,7 @@ def send_cost_report_email( workspace_name=cost_report_vo.workspace_name, report_date=cost_report_vo.issue_date, report_period=self.get_date_range_of_month(cost_report_vo.report_month), - reset_password_link=cost_report_link, + download_link=cost_report_link, ) subject = f'[{service_name}] #{cost_report_vo.report_number} {language_map_info["cost_report"]}' @@ -61,5 +61,5 @@ def _get_service_name(): @staticmethod def get_date_range_of_month(report_month: str): year, month = report_month.split("-") - first_day, last_day = calendar.monthrange(year, month) + first_day, last_day = calendar.monthrange(int(year), int(month)) return f"{year}-{month}-{first_day} ~ {year}-{month}-{last_day}" diff --git a/src/spaceone/cost_analysis/manager/identity_manager.py b/src/spaceone/cost_analysis/manager/identity_manager.py index b25502d1..9dd4b826 100644 --- a/src/spaceone/cost_analysis/manager/identity_manager.py +++ b/src/spaceone/cost_analysis/manager/identity_manager.py @@ -18,14 +18,22 @@ def __init__(self, *args, **kwargs): SpaceConnector, service="identity" ) - def check_workspace(self, workspace_id: str, domain_id: str) -> None: + def get_domain_name(self, domain_id: str) -> str: system_token = config.get_global("TOKEN") self.identity_conn.dispatch( + "Domain.get", {"domain_id": domain_id}, token=system_token + ) + + def check_workspace(self, workspace_id: str, domain_id: str) -> None: + system_token = config.get_global("TOKEN") + + domain_info = self.identity_conn.dispatch( "Workspace.check", {"workspace_id": workspace_id, "domain_id": domain_id}, token=system_token, ) + return domain_info["name"] @cache.cacheable( key="cost-analysis:workspace-name:{domain_id}:{workspace_id}:name", expire=300 diff --git a/src/spaceone/cost_analysis/model/__init__.py b/src/spaceone/cost_analysis/model/__init__.py index 5234eee1..4243873d 100644 --- a/src/spaceone/cost_analysis/model/__init__.py +++ b/src/spaceone/cost_analysis/model/__init__.py @@ -6,3 +6,6 @@ from spaceone.cost_analysis.model.cost_query_set_model import CostQuerySet from spaceone.cost_analysis.model.job_model import Job from spaceone.cost_analysis.model.job_task_model import JobTask +from spaceone.cost_analysis.model.cost_report_config.database import CostReportConfig +from spaceone.cost_analysis.model.cost_report_data.database import CostReportData +from spaceone.cost_analysis.model.cost_report.database import CostReport diff --git a/src/spaceone/cost_analysis/model/cost_report/database.py b/src/spaceone/cost_analysis/model/cost_report/database.py index 9480c2e9..26a8d79d 100644 --- a/src/spaceone/cost_analysis/model/cost_report/database.py +++ b/src/spaceone/cost_analysis/model/cost_report/database.py @@ -31,8 +31,5 @@ class CostReport(MongoModel): "domain_id", ], "ordering": ["created_at"], - "indexes": [ - "domain_id", - "cost_report_id", - ], + "indexes": ["cost_report_config_id"], } diff --git a/src/spaceone/cost_analysis/model/cost_report_data/database.py b/src/spaceone/cost_analysis/model/cost_report_data/database.py index 9c2c887c..e1e84afa 100644 --- a/src/spaceone/cost_analysis/model/cost_report_data/database.py +++ b/src/spaceone/cost_analysis/model/cost_report_data/database.py @@ -35,9 +35,8 @@ class CostReportData(MongoModel): ], "ordering": ["is_confirmed"], "indexes": [ - "domain_id", + "cost_report_config_id", + "cost_report_id", "-report_year", - "cost_report_data_id", - "data_source_id", ], } diff --git a/src/spaceone/cost_analysis/service/cost_report_config_service.py b/src/spaceone/cost_analysis/service/cost_report_config_service.py index 631c18b2..0fb1db68 100644 --- a/src/spaceone/cost_analysis/service/cost_report_config_service.py +++ b/src/spaceone/cost_analysis/service/cost_report_config_service.py @@ -202,12 +202,12 @@ def run(self, params: CostReportConfigRunRequest) -> None: params.cost_report_config_id, params.domain_id ) - cost_report_service = CostReportService() - cost_report_service.create_cost_report(cost_report_config_vo) - cost_report_data_service = CostReportDataService() cost_report_data_service.create_cost_report_data(cost_report_config_vo) + cost_report_service = CostReportService() + cost_report_service.create_cost_report(cost_report_config_vo) + @transaction( permission="cost-analysis:CostReportConfig.read", role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER"], @@ -228,7 +228,7 @@ def get( CostReportConfigResponse: """ cost_report_config_vo = self.cost_report_mgr.get_cost_report_config( - params.cost_report_config_id, params.domain_id, params.workspace_id + params.cost_report_config_id, params.domain_id ) return CostReportConfigResponse(**cost_report_config_vo.to_dict()) diff --git a/src/spaceone/cost_analysis/service/cost_report_data_service.py b/src/spaceone/cost_analysis/service/cost_report_data_service.py index 406e204b..97e27cf6 100644 --- a/src/spaceone/cost_analysis/service/cost_report_data_service.py +++ b/src/spaceone/cost_analysis/service/cost_report_data_service.py @@ -51,16 +51,34 @@ def create_cost_report_data_by_cost_report_config(self, params: dict) -> None: [ "cost_report_config_id", "cost_report_data_id", + "product", + "provider", + "is_confirmed", + "data_source_id", "workspace_id", "domain_id", ] ) + @append_keyword_filter(["product", "cost_report_data_id"]) @convert_model - def stat(self, params: CostReportDataStatQueryRequest) -> dict: - """Analyze cost report data""" + def list( + self, params: CostReportDataSearchQueryRequest + ) -> Union[CostReportsDataResponse, dict]: + """List cost report data""" query = params.query or {} - return self.cost_report_data_mgr.stat_cost_reports_data(query) + ( + cost_report_data_vos, + total_count, + ) = self.cost_report_data_mgr.list_cost_reports_data(query) + + cost_reports_data_info = [ + cost_report_data_vo.to_dict() + for cost_report_data_vo in cost_report_data_vos + ] + return CostReportsDataResponse( + results=cost_reports_data_info, total_count=total_count + ) @transaction( permission="cost-analysis:CostReportData.read", @@ -87,7 +105,10 @@ def analyze(self, params: CostReportDataAnalyzeQueryRequest) -> dict: """Analyze cost report data""" query = params.query or {} - return self.cost_report_data_mgr.analyze_cost_reports_data(query) + + return self.cost_report_data_mgr.analyze_cost_reports_data( + query, target="PRIMARY" + ) @transaction( permission="cost-analysis:CostReportData.read", @@ -97,34 +118,16 @@ def analyze(self, params: CostReportDataAnalyzeQueryRequest) -> dict: [ "cost_report_config_id", "cost_report_data_id", - "product", - "provider", - "is_confirmed", - "data_source_id", "workspace_id", "domain_id", ] ) - @append_keyword_filter(["product", "cost_report_data_id"]) @convert_model - def list( - self, params: CostReportDataSearchQueryRequest - ) -> Union[CostReportsDataResponse, dict]: - """List cost report data""" + def stat(self, params: CostReportDataStatQueryRequest) -> dict: + """Analyze cost report data""" query = params.query or {} - ( - cost_report_data_vos, - total_count, - ) = self.cost_report_data_mgr.list_cost_reports_data(query) - - cost_reports_data_info = [ - cost_report_data_vo.to_dict() - for cost_report_data_vo in cost_report_data_vos - ] - return CostReportsDataResponse( - results=cost_reports_data_info, total_count=total_count - ) + return self.cost_report_data_mgr.stat_cost_reports_data(query) @staticmethod def _get_all_cost_report_configs() -> QuerySet: diff --git a/src/spaceone/cost_analysis/service/cost_report_serivce.py b/src/spaceone/cost_analysis/service/cost_report_serivce.py index 37d7ad14..878cca99 100644 --- a/src/spaceone/cost_analysis/service/cost_report_serivce.py +++ b/src/spaceone/cost_analysis/service/cost_report_serivce.py @@ -8,7 +8,9 @@ import pandas as pd from mongoengine import QuerySet +from spaceone.core import config from spaceone.core.service import * +from spaceone.cost_analysis.model.cost_report.database import CostReport from spaceone.cost_analysis.model.cost_report_config.database import CostReportConfig from spaceone.cost_analysis.model.cost_report.request import * from spaceone.cost_analysis.model.cost_report.response import * @@ -44,6 +46,12 @@ def create_cost_report_by_cost_report_config(self, params: dict): for cost_report_config_vo in self._get_all_cost_report_configs(): self.create_cost_report(cost_report_config_vo) + def send_cost_report_by_cost_report_config(self, params: dict): + """Send cost report by cost report config""" + cost_report_vos = self.cost_report_mgr.filter_cost_reports(status="SUCCESS") + for cost_report_vo in cost_report_vos: + self.send_cost_report(cost_report_vo) + @transaction( permission="cost-analysis:CostReport.read", role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER"], @@ -54,65 +62,41 @@ def send(self, params: CostReportSendRequest) -> None: domain_id = params.domain_id workspace_id = params.workspace_id - cost_report_vo = self.cost_report_mgr.filter_cost_reports( - cost_report_id=params.cost_report_id, - domain_id=domain_id, - workspace_id=workspace_id, - status="SUCCESS", - )[0] - - # Get Cost Report Config - cost_report_config_id = cost_report_vo.cost_report_config_id - cost_report_config_vo = self.cost_report_config_mgr.get_cost_report_config( - cost_report_config_id, domain_id, workspace_id - ) - - recipients = cost_report_config_vo.recipients - role_types = recipients.get("role_types", []) - emails = recipients.get("emails", []) - - # list workspace owner role bindings - identity_mgr = IdentityManager() - - workspace_ids = [] - if workspace_id is not None: - rb_query = { - "filter": [ - {"k": "role_type", "v": role_types, "o": "in"}, - {"k": "workspace_id", "v": workspace_id, "o": "eq"}, - ], - } - role_bindings_info = identity_mgr.list_role_bindings( - params={"query": rb_query}, domain_id=domain_id - ) - - workspace_ids = [ - role_binding_info["workspace_id"] - for role_binding_info in role_bindings_info + query = { + "filter": [ + {"k": "cost_report_id", "v": params.cost_report_id, "o": "eq"}, + {"k": "domain_id", "v": domain_id, "o": "eq"}, + # {"k": "status", "v": "SUCCESS", "o": "eq"}, ] - workspace_ids = list(set(workspace_ids)) - else: - workspace_ids.append(workspace_id) + } + if workspace_id is not None: + query["filter"].append({"k": "workspace_id", "v": workspace_id, "o": "eq"}) - # list workspace owner users - email_mgr = EmailManager() - for ws_id in workspace_ids: - users_info = identity_mgr.list_workspace_users( - params={"workspace_id": ws_id, "state": "ENABLED"}, domain_id=domain_id - ) - for user_info in users_info: - user_id = user_info["user_id"] - email = user_info.get("email", user_id) - email_mgr.send_cost_report_email(user_id, email) + cost_report_vos, total_count = self.cost_report_mgr.list_cost_reports(query) + self.send_cost_report(cost_report_vos[0]) + @transaction( + permission="cost-analysis:CostReport.read", + role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER"], + ) + @convert_model def get_url(self, params: CostReportGetUrlRequest) -> dict: """Get cost report url""" + domain_id = params.domain_id + cost_report_id = params.cost_report_id + + # check cost report cost_report_vo = self.cost_report_mgr.get_cost_report( - params.cost_report_id, params.domain_id, params.workspace_id + domain_id, cost_report_id, params.workspace_id + ) + + # sso_access_token = self._get_temporary_sso_access_token(domain_id) + cost_report_link = self._get_console_cost_report_url( + domain_id, cost_report_id, "token" ) - return {} + return {"cost_report_link": cost_report_link} @transaction( permission="cost-analysis:CostReport.read", @@ -123,7 +107,7 @@ def get(self, params: CostReportGetRequest) -> Union[CostReportResponse, dict]: """Get cost report""" cost_report_vo = self.cost_report_mgr.get_cost_report( - params.cost_report_id, params.domain_id, params.workspace_id + params.domain_id, params.cost_report_id, params.workspace_id ) return CostReportResponse(**cost_report_vo.to_dict()) @@ -155,6 +139,10 @@ def list( query = params.query or {} + # todo: temporary code + # if params.status is None: + # query["filter"].append({"k": "status", "v": "SUCCESS", "o": "eq"}) + cost_report_vos, total_count = self.cost_report_mgr.list_cost_reports(query) cost_reports_info = [ @@ -318,6 +306,81 @@ def _delete_old_cost_reports(self, report_month: str, domain_id: str) -> None: ) cost_reports_vos.delete() + def send_cost_report(self, cost_report_vo: CostReport) -> None: + issue_date = cost_report_vo.issue_date + if issue_date != datetime.utcnow().strftime("%Y-%m-%d"): + _LOGGER.debug( + f"[send_cost_report] skip send cost report ({cost_report_vo.workspace_id}/{cost_report_vo.cost_report_id})" + ) + return + domain_id = cost_report_vo.domain_id + workspace_id = cost_report_vo.workspace_id + + # Get Cost Report Config + cost_report_config_id = cost_report_vo.cost_report_config_id + cost_report_config_vo = self.cost_report_config_mgr.get_cost_report_config( + cost_report_config_id, domain_id + ) + + recipients = cost_report_config_vo.recipients + role_types = recipients.get("role_types", []) + emails = recipients.get("emails", []) + + # list workspace owner role bindings + identity_mgr = IdentityManager() + + workspace_ids = [] + if workspace_id is not None: + rb_query = { + "filter": [ + {"k": "role_type", "v": role_types, "o": "in"}, + {"k": "workspace_id", "v": workspace_id, "o": "eq"}, + ], + } + role_bindings_info = identity_mgr.list_role_bindings( + params={"query": rb_query}, domain_id=domain_id + ) + + workspace_ids = [ + role_binding_info["workspace_id"] + for role_binding_info in role_bindings_info.get("results", []) + ] + workspace_ids = list(set(workspace_ids)) + else: + workspace_ids.append(workspace_id) + + # list workspace owner users + email_mgr = EmailManager() + for workspace_id in workspace_ids: + users_info = identity_mgr.list_workspace_users( + params={"workspace_id": workspace_id, "state": "ENABLED"}, + domain_id=domain_id, + ) + # sso_access_token = self._get_temporary_sso_access_token(domain_id) + cost_report_link = self._get_console_cost_report_url( + domain_id, cost_report_vo.cost_report_id, "token" + ) + for user_info in users_info.get("results", []): + user_id = user_info["user_id"] + email = user_info.get("email", user_id) + language = user_info.get("language", "en") + email_mgr.send_cost_report_email( + user_id, email, cost_report_link, language, cost_report_vo + ) + _LOGGER.debug( + f"[send_cost_report] send cost report ({workspace_id}/{cost_report_vo.cost_report_id}) to {users_info.get('total_count', 0)} users" + ) + + def _get_console_cost_report_url( + self, domain_id: str, cost_report_id: str, token: str + ) -> str: + domain_name = self._get_domain_name(domain_id) + + console_domain = config.get_global("EMAIL_CONSOLE_DOMAIN") + console_domain = console_domain.format(domain_name=domain_name) + + return f"{console_domain}cost-report?sso_access_token={token}&cost_report_id={cost_report_id}" + @staticmethod def _get_current_and_last_month() -> Tuple[str, str]: current_month = datetime.utcnow().strftime("%Y-%m") @@ -403,3 +466,20 @@ def _aggregate_result_by_currency(results: list) -> list: workspace_result_map[workspace_id] = result.copy() return [workspace_result for workspace_result in workspace_result_map.values()] + + @staticmethod + def _get_domain_name(domain_id: str) -> str: + identity_mgr = IdentityManager() + domain_name = identity_mgr.get_domain_name(domain_id) + return domain_name + + def _get_temporary_sso_access_token(self, domain_id: str) -> str: + identity_mgr = IdentityManager() + system_token = config.get_global("TOKEN") + params = { + "grant_type": "SYSTEM_TOKEN", + "scope": "SYSTEM", + } + # todo : make temporary token + token_info = identity_mgr.grant_token({"token_type"}, token=system_token) + return token_info["access_token"]