diff --git a/src/cloudforet/cost_analysis/conf/cost_conf.py b/src/cloudforet/cost_analysis/conf/cost_conf.py index 33219c8..1d9e8b6 100644 --- a/src/cloudforet/cost_analysis/conf/cost_conf.py +++ b/src/cloudforet/cost_analysis/conf/cost_conf.py @@ -9,27 +9,10 @@ "UsageQuantity": {"name": "UsageQuantity", "function": "Sum"}, } -GROUPING = [ - {"type": "Dimension", "name": "ResourceGroup"}, - {"type": "Dimension", "name": "ResourceType"}, - {"type": "Dimension", "name": "ResourceLocation"}, - {"type": "Dimension", "name": "SubscriptionId"}, - {"type": "Dimension", "name": "SubscriptionName"}, - {"type": "Dimension", "name": "MeterCategory"}, - {"type": "Dimension", "name": "Meter"}, - {"type": "Dimension", "name": "UnitOfMeasure"}, - {"type": "Dimension", "name": "BenefitName"}, - {"type": "Dimension", "name": "PricingModel"}, - {"type": "Dimension", "name": "MeterSubcategory"}, -] - GROUPING_EA_AGREEMENT_OPTION = [ {"type": "Dimension", "name": "DepartmentName"}, {"type": "Dimension", "name": "EnrollmentAccountName"}, ] -GROUPING_CUSTOMER_TENANT_OPTION = {"type": "Dimension", "name": "CustomerTenantId"} -GROUPING_TAG_OPTION = {"type": "Tag", "name": ""} -GROUPING_RESOURCE_ID_OPTION = {"type": "Dimension", "name": "ResourceId"} REGION_MAP = { "global": "Global", @@ -68,3 +51,5 @@ "billing_account_id": "providers/Microsoft.Billing/billingAccounts/{billing_account_id}", "customer_tenant_id": "providers/Microsoft.Billing/billingAccounts/{billing_account_id}/customers/{customer_tenant_id}", } + +EXCLUDE_LICENSE_SERVICE_FAMILY = ["Office 365 Global"] diff --git a/src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py b/src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py index 9c115ef..ad7bb2e 100644 --- a/src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py +++ b/src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py @@ -77,36 +77,6 @@ def get_billing_account(self) -> dict: billing_account_info = self.convert_nested_dictionary(billing_account_info) return billing_account_info - def query_http(self, scope, secret_data, parameters, **kwargs): - try: - api_version = "2023-03-01" - self.next_link = f"https://management.azure.com/{scope}/providers/Microsoft.CostManagement/query?api-version={api_version}" - - while self.next_link: - url = self.next_link - - headers = self._make_request_headers( - client_type=secret_data.get("client_id") - ) - response = requests.post(url=url, headers=headers, json=parameters) - response_json = response.json() - - if response_json.get("error"): - response_json = self._retry_request( - response=response, - url=url, - headers=headers, - json=parameters, - retry_count=RETRY_COUNT, - method="post", - **kwargs, - ) - - self.next_link = response_json.get("properties").get("nextLink", None) - yield response_json - except Exception as e: - raise ERROR_UNKNOWN(message=f"[ERROR] query_http {e}") - def begin_create_operation(self, scope: str, parameters: dict) -> list: try: content_type = "application/json" @@ -128,7 +98,7 @@ def begin_create_operation(self, scope: str, parameters: dict) -> list: _LOGGER.error(f"[begin_create_operation] error message: {e}") raise ERROR_UNKNOWN(message=f"[ERROR] begin_create_operation failed") - def get_cost_data(self, blobs: list) -> list: + def get_cost_data(self, blobs: list, options: dict) -> list: for blob in blobs: cost_csv = self._download_cost_data(blob) @@ -137,7 +107,9 @@ def get_cost_data(self, blobs: list) -> list: costs_data = df.to_dict("records") - _LOGGER.debug(f"[get_cost_data] costs count: {len(costs_data)}") + _LOGGER.debug( + f"[get_cost_data] costs count: {len(costs_data)}, options: {options}" + ) # Paginate page_count = int(len(costs_data) / _PAGE_SIZE) + 1 @@ -163,43 +135,6 @@ def _make_request_headers(self, client_type=None): return headers - def _retry_request( - self, response, url, headers, json, retry_count, method="post", **kwargs - ): - try: - _LOGGER.debug(f"{datetime.utcnow()}[INFO] retry_request {response.headers}") - if retry_count == 0: - raise ERROR_UNKNOWN( - message=f"[ERROR] retry_request failed {response.json()}" - ) - elif response.status_code == 400: - raise ERROR_UNKNOWN( - message=f"[ERROR] retry_request failed {response.json()}" - ) - - _sleep_time = self._get_sleep_time(response.headers) - time.sleep(_sleep_time) - - if method == "post": - response = requests.post(url=url, headers=headers, json=json) - else: - response = requests.get(url=url, headers=headers, json=json) - response_json = response.json() - - if response_json.get("error"): - response_json = self._retry_request( - response=response, - url=url, - headers=headers, - json=json, - retry_count=retry_count - 1, - method=method, - ) - return response_json - except Exception as e: - _LOGGER.error(f"[ERROR] retry_request failed {e}") - raise e - def convert_nested_dictionary(self, cloud_svc_object): cloud_svc_dict = {} if hasattr( diff --git a/src/cloudforet/cost_analysis/manager/cost_manager.py b/src/cloudforet/cost_analysis/manager/cost_manager.py index 605ff5b..610f5b7 100644 --- a/src/cloudforet/cost_analysis/manager/cost_manager.py +++ b/src/cloudforet/cost_analysis/manager/cost_manager.py @@ -2,11 +2,12 @@ import logging import json import time - from typing import Union from datetime import datetime, timezone + from spaceone.core.error import * from spaceone.core.manager import BaseManager + from cloudforet.cost_analysis.connector.azure_cost_mgmt_connector import ( AzureCostMgmtConnector, ) @@ -23,7 +24,7 @@ def __init__(self, *args, **kwargs): ) def get_linked_accounts( - self, options: dict, secret_data: dict, domain_id: str, schema + self, options: dict, secret_data: dict, domain_id: str, schema ) -> list: self.azure_cm_connector.create_session(options, secret_data, schema) billing_account_info = self.azure_cm_connector.get_billing_account() @@ -50,7 +51,7 @@ def get_linked_accounts( return accounts_info def get_data( - self, options: dict, secret_data: dict, schema, task_options: dict + self, options: dict, secret_data: dict, schema, task_options: dict ) -> list: self.azure_cm_connector.create_session(options, secret_data, schema) self._check_task_options(task_options) @@ -81,7 +82,7 @@ def get_data( scope, parameters ) - response_stream = self.azure_cm_connector.get_cost_data(blobs) + response_stream = self.azure_cm_connector.get_cost_data(blobs, options) for results in response_stream: yield self._make_cost_data( results=results, end=_end, tenant_id=tenant_id, options=options @@ -96,7 +97,7 @@ def get_data( yield [] def _make_cost_data( - self, results: list, end: datetime, options: dict, tenant_id: str = None + self, results: list, end: datetime, options: dict, tenant_id: str = None ) -> list: """Source Data Model""" @@ -109,7 +110,7 @@ def _make_cost_data( if not billed_date: continue - if self._skip_cost_data_rule(result): + if self._exclude_cost_data_with_options(result, options): continue data = self._make_data_info(result, billed_date, options, tenant_id) @@ -122,7 +123,7 @@ def _make_cost_data( return costs_data def _make_data_info( - self, result: dict, billed_date: str, options: dict, tenant_id: str = None + self, result: dict, billed_date: str, options: dict, tenant_id: str = None ): additional_info = self._get_additional_info(result, options, tenant_id) cost = self._get_cost_from_result_with_options(result, options) @@ -187,8 +188,8 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non additional_info["Benefit Name"] = benefit_name if ( - result.get("pricingmodel") == "Reservation" - and result["metercategory"] == "" + result.get("pricingmodel") == "Reservation" + and result["metercategory"] == "" ): result["metercategory"] = self._set_product_from_benefit_name( benefit_name @@ -199,14 +200,14 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non if result.get("metersubcategory") != "" and result.get("metersubcategory"): additional_info["Meter SubCategory"] = result.get("metersubcategory") if ( - result.get("pricingmodel") == "OnDemand" - and result.get("metercategory") == "" + result.get("pricingmodel") == "OnDemand" + and result.get("metercategory") == "" ): result["metercategory"] = result.get("metercategory") if result.get("customername") is None: if result.get("invoicesectionname") != "" and result.get( - "invoicesectionname" + "invoicesectionname" ): additional_info["Department Name"] = result.get("invoicesectionname") elif result.get("departmentname") != "" and result.get("departmentname"): @@ -215,15 +216,15 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non if result.get("accountname") != "" and result.get("accountname"): additional_info["Enrollment Account Name"] = result["accountname"] elif result.get("enrollmentaccountname") != "" and result.get( - "enrollmentaccountname" + "enrollmentaccountname" ): additional_info["Enrollment Account Name"] = result["enrollmentaccountname"] collect_resource_id = options.get("collect_resource_id", False) if ( - collect_resource_id - and result.get("resourceid") != "" - and result.get("resourceid") + collect_resource_id + and result.get("resourceid") != "" + and result.get("resourceid") ): additional_info["Resource Id"] = result["resourceid"] additional_info["Resource Name"] = result["resourceid"].split("/")[-1] @@ -307,10 +308,10 @@ def _get_tenant_ids(task_options: dict, collect_scope: str) -> list: @staticmethod def _make_scope( - secret_data: dict, - task_options: dict, - collect_scope: str, - customer_tenant_id: str = None, + secret_data: dict, + task_options: dict, + collect_scope: str, + customer_tenant_id: str = None, ): if collect_scope == "subscription_id": subscription_id = task_options["subscription_id"] @@ -411,7 +412,7 @@ def _convert_date_format_to_utc(date_format: str) -> datetime: return datetime.strptime(date_format, "%Y-%m-%d").replace(tzinfo=timezone.utc) def _make_monthly_time_period( - self, start_date: datetime, end_date: datetime + self, start_date: datetime, end_date: datetime ) -> list: monthly_time_period = [] current_date = end_date @@ -441,7 +442,7 @@ def _make_monthly_time_period( @staticmethod def _get_linked_customer_tenants( - secret_data: dict, billing_accounts_info: list + secret_data: dict, billing_accounts_info: list ) -> list: customer_tenants = secret_data.get("customer_tenants", []) if not customer_tenants: @@ -454,7 +455,7 @@ def _get_linked_customer_tenants( @staticmethod def _make_accounts_info_from_customer_tenants( - billing_accounts_info: list, customer_tenants: list + billing_accounts_info: list, customer_tenants: list ) -> list: accounts_info = [] for billing_account_info in billing_accounts_info: @@ -481,14 +482,18 @@ def _check_task_options(task_options): raise ERROR_REQUIRED_PARAMETER(key="task_options.customer_tenants") @staticmethod - def _skip_cost_data_rule(result: dict) -> bool: - if result.get("customertenentname") and not result.get("customertenantid"): + def _exclude_cost_data_with_options(result: dict, options: dict) -> bool: + if result.get("customername") and not result.get("customertenantid"): return True + if options.get("exclude_license_cost", False): + if result.get("servicefamily") in EXCLUDE_LICENSE_SERVICE_FAMILY: + return True + return False @staticmethod def _set_network_traffic_cost( - additional_info: dict, product: str, usage_type: str + additional_info: dict, product: str, usage_type: str ) -> dict: if product in ["Bandwidth", "Content Delivery Network"]: additional_info["Usage Type Details"] = usage_type diff --git a/src/cloudforet/cost_analysis/manager/data_source_manager.py b/src/cloudforet/cost_analysis/manager/data_source_manager.py index 7204299..e19eab9 100644 --- a/src/cloudforet/cost_analysis/manager/data_source_manager.py +++ b/src/cloudforet/cost_analysis/manager/data_source_manager.py @@ -27,6 +27,9 @@ def init_response(options): plugin_metadata.use_account_routing = True plugin_metadata.account_match_key = "additional_info.Tenant Id" + if options.get("exclude_license_cost"): + plugin_metadata.exclude_license_cost = True + plugin_metadata.validate() return {"metadata": plugin_metadata.to_primitive()} diff --git a/src/cloudforet/cost_analysis/model/data_source_model.py b/src/cloudforet/cost_analysis/model/data_source_model.py index 0c66467..4248162 100644 --- a/src/cloudforet/cost_analysis/model/data_source_model.py +++ b/src/cloudforet/cost_analysis/model/data_source_model.py @@ -63,3 +63,4 @@ class PluginMetadata(Model): use_account_routing = BooleanType(default=False) alias = DictType(StringType, default={}) account_match_key = StringType(default=None) + exclude_license_cost = BooleanType(default=False)