From 1731e8dddf90bd65394486fd53d7a4700e1f6781 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Mon, 4 Dec 2023 11:40:19 +0530 Subject: [PATCH] Changed the cost reports strategy --- .github/workflows/Build.yml | 1 + .github/workflows/PR.yml | 1 + .../cost_management_operations.py | 5 +++- .../azure/subscriptions/azure_operations.py | 19 +++++++++++++++ .../policy/azure/cost_billing_reports.py | 24 +++++++++++-------- .../test_cost_management_operations.py | 13 ++++++---- 6 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 4544a957..8ba2e23f 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -171,6 +171,7 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_ACCOUNT_ID: ${{ secrets.AZURE_ACCOUNT_ID }} GCP_DATABASE_NAME: ${{ secrets.GCP_DATABASE_NAME }} GCP_DATABASE_TABLE_NAME: ${{ secrets.GCP_DATABASE_TABLE_NAME }} run: | diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 6ac2d3eb..b9857103 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -174,6 +174,7 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_ACCOUNT_ID: ${{ secrets.AZURE_ACCOUNT_ID }} GCP_DATABASE_NAME: ${{ secrets.GCP_DATABASE_NAME }} GCP_DATABASE_TABLE_NAME: ${{ secrets.GCP_DATABASE_TABLE_NAME }} run: | diff --git a/cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py b/cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py index 1d368f5c..b4591c9b 100644 --- a/cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py +++ b/cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py @@ -43,7 +43,10 @@ def __get_query_dataset(self, grouping: list, tags: dict, granularity: str): if grouping: filter_grouping = [] for group in grouping: - filter_grouping.append({"name": group.lower(), "type": "TagKey"}) + if isinstance(group, dict): + filter_grouping.append({"name": group['name'], "type": group['type']}) + else: + filter_grouping.append({"name": group.lower(), "type": "TagKey"}) query_dataset['grouping'] = filter_grouping return query_dataset diff --git a/cloud_governance/common/clouds/azure/subscriptions/azure_operations.py b/cloud_governance/common/clouds/azure/subscriptions/azure_operations.py index d51801c8..c27be615 100644 --- a/cloud_governance/common/clouds/azure/subscriptions/azure_operations.py +++ b/cloud_governance/common/clouds/azure/subscriptions/azure_operations.py @@ -49,6 +49,25 @@ def get_billing_profiles(self): response = self.billing_client.billing_profiles.list_by_billing_account(self.__account_id) return response + def get_billing_profiles_list(self, subscription_id: str = ''): + """ + This method returns list of billing profiles + :param subscription_id: + :type subscription_id: + :return: + :rtype: + """ + billing_profiles = [] + responses = self.billing_client.billing_profiles.list_by_billing_account(self.__account_id) + for response in responses: + if subscription_id: + if subscription_id == response.subscriptionId: + return response.id + else: + billing_profiles.append(response.id) + return billing_profiles + + diff --git a/cloud_governance/policy/azure/cost_billing_reports.py b/cloud_governance/policy/azure/cost_billing_reports.py index ad7c11fa..39e6c6f9 100644 --- a/cloud_governance/policy/azure/cost_billing_reports.py +++ b/cloud_governance/policy/azure/cost_billing_reports.py @@ -8,6 +8,7 @@ from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload from cloud_governance.common.google_drive.google_drive_operations import GoogleDriveOperations from cloud_governance.common.google_drive.upload_to_gsheet import UploadToGsheet +from cloud_governance.common.logger.init_logger import logger from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp from cloud_governance.main.environment_variables import environment_variables @@ -131,8 +132,8 @@ def get_total_billing_accounts(self): cost_center = int(profile.get('display_name').split('(CC ')[-1].split(')')[0]) self.__common_data['CostCenter'] = cost_center filters = { - 'grouping': [QueryGrouping(type='Dimension', name='SubscriptionName'), - QueryGrouping(type='Dimension', name='SubscriptionId')] + 'grouping': [{'type':'Dimension', 'name': 'SubscriptionName'}, + {'type': 'Dimension', 'name': 'SubscriptionId'}] } usage_data = self.cost_mgmt_operations.get_usage(scope=scope, **filters) subscription_ids = [] @@ -142,8 +143,7 @@ def get_total_billing_accounts(self): cost_billing_data=cost_billing_data) for subscription in subscription_ids: query_filter = { - 'filter': QueryFilter(dimensions=QueryComparisonExpression(name='SubscriptionId', operator='in', - values=[subscription[0]]))} + 'filter': {'name': 'SubscriptionId', 'values': [subscription[0]]}} forecast_data = self.cost_mgmt_operations.get_forecast(scope=scope, **query_filter) if forecast_data and forecast_data.get('rows'): self.get_data_from_costs(cost_data_rows=forecast_data.get('rows'), @@ -161,12 +161,16 @@ def collect_and_upload_cost_data(self): """ cost_billing_data = {} if not self.__total_account: - usage_data = self.cost_mgmt_operations.get_usage(scope=self.azure_operations.scope) - forecast_data = self.cost_mgmt_operations.get_forecast(scope=self.azure_operations.scope) - if usage_data: - self.get_data_from_costs(cost_data_rows=usage_data.get('rows'), cost_data_columns=usage_data.get('columns'), cost_type='Actual', cost_billing_data=cost_billing_data) - if forecast_data: - self.get_data_from_costs(cost_data_rows=forecast_data.get('rows'), cost_data_columns=forecast_data.get('columns'), cost_type='Forecast', cost_billing_data=cost_billing_data, subscription_id=self.azure_operations.subscription_id) + scope = self.azure_operations.get_billing_profiles_list(subscription_id=self.azure_operations.subscription_id) + if scope: + usage_data = self.cost_mgmt_operations.get_usage(scope=scope) + forecast_data = self.cost_mgmt_operations.get_forecast(scope=self.azure_operations.scope) + if usage_data: + self.get_data_from_costs(cost_data_rows=usage_data.get('rows'), cost_data_columns=usage_data.get('columns'), cost_type='Actual', cost_billing_data=cost_billing_data) + if forecast_data: + self.get_data_from_costs(cost_data_rows=forecast_data.get('rows'), cost_data_columns=forecast_data.get('columns'), cost_type='Forecast', cost_billing_data=cost_billing_data, subscription_id=self.azure_operations.subscription_id) + else: + logger.warning(f"No billing scopes found in the subscription: {self.azure_operations.subscription_id}") else: cost_billing_data = self.get_total_billing_accounts() if cost_billing_data: diff --git a/tests/integration/cloud_governance/common/clouds/azure/cost_management/test_cost_management_operations.py b/tests/integration/cloud_governance/common/clouds/azure/cost_management/test_cost_management_operations.py index 9a6eb190..c6c99b2e 100644 --- a/tests/integration/cloud_governance/common/clouds/azure/cost_management/test_cost_management_operations.py +++ b/tests/integration/cloud_governance/common/clouds/azure/cost_management/test_cost_management_operations.py @@ -14,11 +14,14 @@ def test_get_usage(): end_date = datetime.datetime.utcnow() - datetime.timedelta(days=2) start_date = end_date - datetime.timedelta(days=1) granularity = 'Daily' - cost_usage_data = cost_management_operations.get_usage(scope=cost_management_operations.azure_operations.scope, - start_date=start_date, end_date=end_date, - granularity=granularity - ) - assert cost_usage_data + scope = cost_management_operations.azure_operations.get_billing_profiles_list()[0] + if scope: + cost_usage_data = cost_management_operations.get_usage(scope=scope, + start_date=start_date, end_date=end_date, + granularity=granularity + ) + assert cost_usage_data + assert False def test_get_forecast():