feat: add exclude license cost data option

Signed-off-by: ImMin5 <[email protected]>

src/cloudforet/cost_analysis/conf/cost_conf.py

@@ -9,27 +9,10 @@
     "UsageQuantity": {"name": "UsageQuantity", "function": "Sum"},
 }
 
-GROUPING = [
-    {"type": "Dimension", "name": "ResourceGroup"},
-    {"type": "Dimension", "name": "ResourceType"},
-    {"type": "Dimension", "name": "ResourceLocation"},
-    {"type": "Dimension", "name": "SubscriptionId"},
-    {"type": "Dimension", "name": "SubscriptionName"},
-    {"type": "Dimension", "name": "MeterCategory"},
-    {"type": "Dimension", "name": "Meter"},
-    {"type": "Dimension", "name": "UnitOfMeasure"},
-    {"type": "Dimension", "name": "BenefitName"},
-    {"type": "Dimension", "name": "PricingModel"},
-    {"type": "Dimension", "name": "MeterSubcategory"},
-]
-
 GROUPING_EA_AGREEMENT_OPTION = [
     {"type": "Dimension", "name": "DepartmentName"},
     {"type": "Dimension", "name": "EnrollmentAccountName"},
 ]
-GROUPING_CUSTOMER_TENANT_OPTION = {"type": "Dimension", "name": "CustomerTenantId"}
-GROUPING_TAG_OPTION = {"type": "Tag", "name": ""}
-GROUPING_RESOURCE_ID_OPTION = {"type": "Dimension", "name": "ResourceId"}
 
 REGION_MAP = {
     "global": "Global",
@@ -68,3 +51,5 @@
     "billing_account_id": "providers/Microsoft.Billing/billingAccounts/{billing_account_id}",
     "customer_tenant_id": "providers/Microsoft.Billing/billingAccounts/{billing_account_id}/customers/{customer_tenant_id}",
 }
+
+EXCLUDE_LICENSE_SERVICE_FAMILY = ["Office 365 Global"]

src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py

@@ -77,36 +77,6 @@ def get_billing_account(self) -> dict:
         billing_account_info = self.convert_nested_dictionary(billing_account_info)
         return billing_account_info
 
-    def query_http(self, scope, secret_data, parameters, **kwargs):
-        try:
-            api_version = "2023-03-01"
-            self.next_link = f"https://management.azure.com/{scope}/providers/Microsoft.CostManagement/query?api-version={api_version}"
-
-            while self.next_link:
-                url = self.next_link
-
-                headers = self._make_request_headers(
-                    client_type=secret_data.get("client_id")
-                )
-                response = requests.post(url=url, headers=headers, json=parameters)
-                response_json = response.json()
-
-                if response_json.get("error"):
-                    response_json = self._retry_request(
-                        response=response,
-                        url=url,
-                        headers=headers,
-                        json=parameters,
-                        retry_count=RETRY_COUNT,
-                        method="post",
-                        **kwargs,
-                    )
-
-                self.next_link = response_json.get("properties").get("nextLink", None)
-                yield response_json
-        except Exception as e:
-            raise ERROR_UNKNOWN(message=f"[ERROR] query_http {e}")
-
     def begin_create_operation(self, scope: str, parameters: dict) -> list:
         try:
             content_type = "application/json"
@@ -128,7 +98,7 @@ def begin_create_operation(self, scope: str, parameters: dict) -> list:
             _LOGGER.error(f"[begin_create_operation] error message: {e}")
             raise ERROR_UNKNOWN(message=f"[ERROR] begin_create_operation failed")
 
-    def get_cost_data(self, blobs: list) -> list:
+    def get_cost_data(self, blobs: list, options: dict) -> list:
         for blob in blobs:
             cost_csv = self._download_cost_data(blob)
 
@@ -137,7 +107,9 @@ def get_cost_data(self, blobs: list) -> list:
 
             costs_data = df.to_dict("records")
 
-            _LOGGER.debug(f"[get_cost_data] costs count: {len(costs_data)}")
+            _LOGGER.debug(
+                f"[get_cost_data] costs count: {len(costs_data)}, options: {options}"
+            )
 
             # Paginate
             page_count = int(len(costs_data) / _PAGE_SIZE) + 1
@@ -163,43 +135,6 @@ def _make_request_headers(self, client_type=None):
 
         return headers
 
-    def _retry_request(
-        self, response, url, headers, json, retry_count, method="post", **kwargs
-    ):
-        try:
-            _LOGGER.debug(f"{datetime.utcnow()}[INFO] retry_request {response.headers}")
-            if retry_count == 0:
-                raise ERROR_UNKNOWN(
-                    message=f"[ERROR] retry_request failed {response.json()}"
-                )
-            elif response.status_code == 400:
-                raise ERROR_UNKNOWN(
-                    message=f"[ERROR] retry_request failed {response.json()}"
-                )
-
-            _sleep_time = self._get_sleep_time(response.headers)
-            time.sleep(_sleep_time)
-
-            if method == "post":
-                response = requests.post(url=url, headers=headers, json=json)
-            else:
-                response = requests.get(url=url, headers=headers, json=json)
-            response_json = response.json()
-
-            if response_json.get("error"):
-                response_json = self._retry_request(
-                    response=response,
-                    url=url,
-                    headers=headers,
-                    json=json,
-                    retry_count=retry_count - 1,
-                    method=method,
-                )
-            return response_json
-        except Exception as e:
-            _LOGGER.error(f"[ERROR] retry_request failed {e}")
-            raise e
-
     def convert_nested_dictionary(self, cloud_svc_object):
         cloud_svc_dict = {}
         if hasattr(

src/cloudforet/cost_analysis/manager/cost_manager.py

@@ -2,11 +2,12 @@
 import logging
 import json
 import time
-
 from typing import Union
 from datetime import datetime, timezone
+
 from spaceone.core.error import *
 from spaceone.core.manager import BaseManager
+
 from cloudforet.cost_analysis.connector.azure_cost_mgmt_connector import (
     AzureCostMgmtConnector,
 )
@@ -23,7 +24,7 @@ def __init__(self, *args, **kwargs):
         )
 
     def get_linked_accounts(
-            self, options: dict, secret_data: dict, domain_id: str, schema
+        self, options: dict, secret_data: dict, domain_id: str, schema
     ) -> list:
         self.azure_cm_connector.create_session(options, secret_data, schema)
         billing_account_info = self.azure_cm_connector.get_billing_account()
@@ -50,7 +51,7 @@ def get_linked_accounts(
         return accounts_info
 
     def get_data(
-            self, options: dict, secret_data: dict, schema, task_options: dict
+        self, options: dict, secret_data: dict, schema, task_options: dict
     ) -> list:
         self.azure_cm_connector.create_session(options, secret_data, schema)
         self._check_task_options(task_options)
@@ -81,7 +82,7 @@ def get_data(
                     scope, parameters
                 )
 
-                response_stream = self.azure_cm_connector.get_cost_data(blobs)
+                response_stream = self.azure_cm_connector.get_cost_data(blobs, options)
                 for results in response_stream:
                     yield self._make_cost_data(
                         results=results, end=_end, tenant_id=tenant_id, options=options
@@ -96,7 +97,7 @@ def get_data(
         yield []
 
     def _make_cost_data(
-            self, results: list, end: datetime, options: dict, tenant_id: str = None
+        self, results: list, end: datetime, options: dict, tenant_id: str = None
     ) -> list:
         """Source Data Model"""
 
@@ -109,7 +110,7 @@ def _make_cost_data(
                 if not billed_date:
                     continue
 
-                if self._skip_cost_data_rule(result):
+                if self._exclude_cost_data_with_options(result, options):
                     continue
 
                 data = self._make_data_info(result, billed_date, options, tenant_id)
@@ -122,7 +123,7 @@ def _make_cost_data(
         return costs_data
 
     def _make_data_info(
-            self, result: dict, billed_date: str, options: dict, tenant_id: str = None
+        self, result: dict, billed_date: str, options: dict, tenant_id: str = None
     ):
         additional_info = self._get_additional_info(result, options, tenant_id)
         cost = self._get_cost_from_result_with_options(result, options)
@@ -187,8 +188,8 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
             additional_info["Benefit Name"] = benefit_name
 
             if (
-                    result.get("pricingmodel") == "Reservation"
-                    and result["metercategory"] == ""
+                result.get("pricingmodel") == "Reservation"
+                and result["metercategory"] == ""
             ):
                 result["metercategory"] = self._set_product_from_benefit_name(
                     benefit_name
@@ -199,14 +200,14 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
         if result.get("metersubcategory") != "" and result.get("metersubcategory"):
             additional_info["Meter SubCategory"] = result.get("metersubcategory")
             if (
-                    result.get("pricingmodel") == "OnDemand"
-                    and result.get("metercategory") == ""
+                result.get("pricingmodel") == "OnDemand"
+                and result.get("metercategory") == ""
             ):
                 result["metercategory"] = result.get("metercategory")
 
         if result.get("customername") is None:
             if result.get("invoicesectionname") != "" and result.get(
-                    "invoicesectionname"
+                "invoicesectionname"
             ):
                 additional_info["Department Name"] = result.get("invoicesectionname")
             elif result.get("departmentname") != "" and result.get("departmentname"):
@@ -215,15 +216,15 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
         if result.get("accountname") != "" and result.get("accountname"):
             additional_info["Enrollment Account Name"] = result["accountname"]
         elif result.get("enrollmentaccountname") != "" and result.get(
-                "enrollmentaccountname"
+            "enrollmentaccountname"
         ):
             additional_info["Enrollment Account Name"] = result["enrollmentaccountname"]
 
         collect_resource_id = options.get("collect_resource_id", False)
         if (
-                collect_resource_id
-                and result.get("resourceid") != ""
-                and result.get("resourceid")
+            collect_resource_id
+            and result.get("resourceid") != ""
+            and result.get("resourceid")
         ):
             additional_info["Resource Id"] = result["resourceid"]
             additional_info["Resource Name"] = result["resourceid"].split("/")[-1]
@@ -307,10 +308,10 @@ def _get_tenant_ids(task_options: dict, collect_scope: str) -> list:
 
     @staticmethod
     def _make_scope(
-            secret_data: dict,
-            task_options: dict,
-            collect_scope: str,
-            customer_tenant_id: str = None,
+        secret_data: dict,
+        task_options: dict,
+        collect_scope: str,
+        customer_tenant_id: str = None,
     ):
         if collect_scope == "subscription_id":
             subscription_id = task_options["subscription_id"]
@@ -411,7 +412,7 @@ def _convert_date_format_to_utc(date_format: str) -> datetime:
         return datetime.strptime(date_format, "%Y-%m-%d").replace(tzinfo=timezone.utc)
 
     def _make_monthly_time_period(
-            self, start_date: datetime, end_date: datetime
+        self, start_date: datetime, end_date: datetime
     ) -> list:
         monthly_time_period = []
         current_date = end_date
@@ -441,7 +442,7 @@ def _make_monthly_time_period(
 
     @staticmethod
     def _get_linked_customer_tenants(
-            secret_data: dict, billing_accounts_info: list
+        secret_data: dict, billing_accounts_info: list
     ) -> list:
         customer_tenants = secret_data.get("customer_tenants", [])
         if not customer_tenants:
@@ -454,7 +455,7 @@ def _get_linked_customer_tenants(
 
     @staticmethod
     def _make_accounts_info_from_customer_tenants(
-            billing_accounts_info: list, customer_tenants: list
+        billing_accounts_info: list, customer_tenants: list
     ) -> list:
         accounts_info = []
         for billing_account_info in billing_accounts_info:
@@ -481,14 +482,18 @@ def _check_task_options(task_options):
             raise ERROR_REQUIRED_PARAMETER(key="task_options.customer_tenants")
 
     @staticmethod
-    def _skip_cost_data_rule(result: dict) -> bool:
-        if result.get("customertenentname") and not result.get("customertenantid"):
+    def _exclude_cost_data_with_options(result: dict, options: dict) -> bool:
+        if result.get("customername") and not result.get("customertenantid"):
             return True
+        if options.get("exclude_license_cost", False):
+            if result.get("servicefamily") in EXCLUDE_LICENSE_SERVICE_FAMILY:
+                return True
+
         return False
 
     @staticmethod
     def _set_network_traffic_cost(
-            additional_info: dict, product: str, usage_type: str
+        additional_info: dict, product: str, usage_type: str
     ) -> dict:
         if product in ["Bandwidth", "Content Delivery Network"]:
             additional_info["Usage Type Details"] = usage_type

src/cloudforet/cost_analysis/manager/data_source_manager.py

@@ -27,6 +27,9 @@ def init_response(options):
             plugin_metadata.use_account_routing = True
             plugin_metadata.account_match_key = "additional_info.Tenant Id"
 
+        if options.get("exclude_license_cost"):
+            plugin_metadata.exclude_license_cost = True
+
         plugin_metadata.validate()
         return {"metadata": plugin_metadata.to_primitive()}
 

src/cloudforet/cost_analysis/model/data_source_model.py

@@ -63,3 +63,4 @@ class PluginMetadata(Model):
     use_account_routing = BooleanType(default=False)
     alias = DictType(StringType, default={})
     account_match_key = StringType(default=None)
+    exclude_license_cost = BooleanType(default=False)