From feb9138bd64356a0d7ae1a3238a8025275cbcff0 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti <97395760+athiruma@users.noreply.github.com> Date: Tue, 17 Oct 2023 17:35:33 +0530 Subject: [PATCH] Added the cro reports to es (#685) --- .../clouds/aws/ec2/aws_monitor_tickets.py | 23 -- .../clouds/aws/ec2/cost_over_usage.py | 2 +- .../clouds/azure/azure_run_cro.py | 37 ---- .../resource_groups/azure_monitor_tickets.py | 66 ++++++ .../azure_tagging_operations.py | 40 ++++ .../resource_groups/collect_cro_reports.py | 208 ++++++++++++++++++ .../azure/resource_groups/cost_over_usage.py | 18 +- .../resource_groups/monitor_cro_resources.py | 76 +++++-- .../resource_groups/tag_cro_resources.py | 2 + .../common/abstract_collect_cro_reports.py | 157 +++++++++++++ .../clouds/common/abstract_cost_over_usage.py | 12 +- .../common/cro_object.py | 108 +++++++++ .../common/run_cro.py | 111 ++++++++++ .../monitor/cloud_monitor.py | 23 +- .../clouds/azure/compute/common_operations.py | 25 ++- .../compute/resource_group_operations.py | 4 +- .../cost_management_operations.py | 77 +++++-- 17 files changed, 852 insertions(+), 137 deletions(-) delete mode 100644 cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py create mode 100644 cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/azure_monitor_tickets.py create mode 100644 cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/azure_tagging_operations.py create mode 100644 cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/collect_cro_reports.py create mode 100644 cloud_governance/cloud_resource_orchestration/clouds/common/abstract_collect_cro_reports.py create mode 100644 cloud_governance/cloud_resource_orchestration/common/cro_object.py create mode 100644 cloud_governance/cloud_resource_orchestration/common/run_cro.py diff --git a/cloud_governance/cloud_resource_orchestration/clouds/aws/ec2/aws_monitor_tickets.py b/cloud_governance/cloud_resource_orchestration/clouds/aws/ec2/aws_monitor_tickets.py index 2de35ae9..6452dbee 100644 --- a/cloud_governance/cloud_resource_orchestration/clouds/aws/ec2/aws_monitor_tickets.py +++ b/cloud_governance/cloud_resource_orchestration/clouds/aws/ec2/aws_monitor_tickets.py @@ -129,29 +129,6 @@ def __send_ticket_status_alerts(self, tickets: dict, ticket_status: str): filename.flush() self.__postfix.send_email_postfix(to=to, cc=cc, subject=subject, content=body, mime_type='html', filename=filename.name) - @typeguard.typechecked - @logger_time_stamp - def verify_es_instances_state(self, es_data: dict): - """ - This method verify the state of the es_instances - :param es_data: - :return: - """ - instance_ids = [resource.split(',')[1].strip() for resource in es_data.get('instances', []) if 'terminated' not in resource] - es_data_change = False - if instance_ids: - local_ec2_operations = EC2Operations(region=self.__region_name) - instances = local_ec2_operations.get_ec2_instance_ids(Filters=[{'Name': 'instance-id', 'Values': instance_ids}]) - instance_ids = list(set(instance_ids) - set(instances)) - for idx, resource in enumerate(es_data.get('instances')): - resource_data = resource.split(',') - instance_id = resource_data[1].strip() - if instance_id in instance_ids: - es_data_change = True - resource_data[4] = 'terminated' - es_data['instances'][idx] = ', '.join(resource_data) - return es_data_change - @logger_time_stamp def __track_tickets(self): """ diff --git a/cloud_governance/cloud_resource_orchestration/clouds/aws/ec2/cost_over_usage.py b/cloud_governance/cloud_resource_orchestration/clouds/aws/ec2/cost_over_usage.py index 188f5528..385022bc 100644 --- a/cloud_governance/cloud_resource_orchestration/clouds/aws/ec2/cost_over_usage.py +++ b/cloud_governance/cloud_resource_orchestration/clouds/aws/ec2/cost_over_usage.py @@ -20,7 +20,7 @@ class CostOverUsage: """ - This class will monitors the cost explorer reports and sends alert to the user if they exceed specified amount + This class monitors the cost explorer reports and sends alert to the user if they exceed specified amount """ DEFAULT_ROUND_DIGITS = 3 diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py deleted file mode 100644 index c63a85a2..00000000 --- a/cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py +++ /dev/null @@ -1,37 +0,0 @@ -from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.cost_over_usage import CostOverUsage -from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.monitor_cro_resources import \ - MonitorCROResources -from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.tag_cro_resources import TagCROResources - - -class AzureRunCro: - - def __init__(self): - pass - - def __run_cloud_resources(self): - """ - This method run the azure resources in specified region or all regions - :return: - """ - CostOverUsage().run() - TagCROResources().run() - monitored_resources = MonitorCROResources().run() - - def __start_cro(self): - """ - This method start the cro process methods - 1. Send alert to cost over usage users - 2. Tag the new instances - 3. monitor and upload the new instances' data - 4. Monitor the Jira ticket progressing - :return: - """ - self.__run_cloud_resources() - - def run(self): - """ - This method start the Azure CRO operations - :return: - """ - self.__start_cro() diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/azure_monitor_tickets.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/azure_monitor_tickets.py new file mode 100644 index 00000000..7156ddfb --- /dev/null +++ b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/azure_monitor_tickets.py @@ -0,0 +1,66 @@ + + +import json +import tempfile +from abc import ABC +from datetime import datetime + +import typeguard + +from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.aws_tagging_operations import AWSTaggingOperations +from cloud_governance.cloud_resource_orchestration.common.abstract_monitor_tickets import AbstractMonitorTickets +from cloud_governance.cloud_resource_orchestration.utils.common_operations import get_tag_value_by_name +from cloud_governance.common.clouds.aws.athena.pyathena_operations import PyAthenaOperations +from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations +from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations +from cloud_governance.common.jira.jira_operations import JiraOperations +from cloud_governance.common.ldap.ldap_search import LdapSearch +from cloud_governance.common.logger.init_logger import handler, logger +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp +from cloud_governance.common.mails.mail_message import MailMessage +from cloud_governance.common.mails.postfix import Postfix +from cloud_governance.main.environment_variables import environment_variables + + +class AzureMonitorTickets(AbstractMonitorTickets): + """This method monitors the Jira Tickets""" + + NEW = 'New' + REFINEMENT = 'Refinement' + CLOSED = 'Closed' + IN_PROGRESS = 'In Progress' + CLOSE_JIRA_TICKET = 0 + FIRST_CRO_ALERT: int = 5 + SECOND_CRO_ALERT: int = 3 + DEFAULT_ROUND_DIGITS: int = 3 + + def __init__(self, region_name: str = ''): + super().__init__() + + # Todo All the below methods implement in future releases + def update_budget_tag_to_resources(self, region_name: str, ticket_id: str, updated_budget: int): + pass + + def update_duration_tag_to_resources(self, region_name: str, ticket_id: str, updated_duration: int): + pass + + def update_cluster_cost(self): + pass + + def extend_tickets_budget(self, ticket_id: str, region_name: str, current_budget: int = 0): + return super().extend_tickets_budget(ticket_id, region_name, current_budget) + + def extend_ticket_duration(self, ticket_id: str, region_name: str, current_duration: int = 0): + return super().extend_ticket_duration(ticket_id, region_name, current_duration) + + @logger_time_stamp + def run(self): + """ + This method run all methods of jira tickets monitoring + :return: + # """ + self.monitor_tickets() + + + + diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/azure_tagging_operations.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/azure_tagging_operations.py new file mode 100644 index 00000000..23f39f31 --- /dev/null +++ b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/azure_tagging_operations.py @@ -0,0 +1,40 @@ + + +import typeguard + +from cloud_governance.cloud_resource_orchestration.clouds.common.abstract_tagging_operations import \ + AbstractTaggingOperations +from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp + + +class AzureTaggingOperations(AbstractTaggingOperations): + """ + This class performs the tagging operations on AWS + """ + + def __init__(self): + super(AbstractTaggingOperations).__init__() + self.__resource_group_operations = ResourceGroupOperations() + + @logger_time_stamp + def tag_resources_list(self, resources_list: list, update_tags_dict: dict): + """ + This method updates the tags to the resources + :param resources_list: + :param update_tags_dict: + :return: + """ + pass + + @typeguard.typechecked + @logger_time_stamp + def get_resources_list(self, tag_name: str, tag_value: str = ''): + """ + This method returns all the resources having the tag_name and tag_value + :param tag_name: + :param tag_value: + :return: + """ + filter_values = f"``$filter=tagName eq '{tag_name}' and tagValue eq '{tag_value}'``" + return self.__resource_group_operations.get_all_resources(filter=filter_values) diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/collect_cro_reports.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/collect_cro_reports.py new file mode 100644 index 00000000..a44cb132 --- /dev/null +++ b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/collect_cro_reports.py @@ -0,0 +1,208 @@ +import logging +from datetime import datetime, timedelta + +import typeguard + +from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.cost_over_usage import CostOverUsage +from cloud_governance.cloud_resource_orchestration.clouds.common.abstract_collect_cro_reports import \ + AbstractCollectCROReports +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp +from cloud_governance.main.environment_variables import environment_variables + + +class CollectCROReports(AbstractCollectCROReports): + """ + This method collects the user/instance-id data from the cost-explorer + """ + + def __init__(self): + super().__init__() + self.__cost_over_usage = CostOverUsage() + self._account_id = self._environment_variables_dict.get('AZURE_SUBSCRIPTION_ID') + self.__scope = f'subscriptions/{self._account_id}' + + def _get_account_budget_from_payer_ce_report(self): + """ + This method returns the account budget from the payer ce reports + Check policy cost_explorer_payer_billings + :return: + """ + query = { + "query": { + "bool": { + "must": [ + {"term": {"CloudName.keyword": self._public_cloud_name}}, + {"term": {"AccountId.keyword": self._account_id}}, + {"term": {"Month": str(datetime.utcnow().year)}}, + ] + } + }, + "size": 1 + } + response = self._es_operations.fetch_data_by_es_query(query=query, es_index=self._ce_payer_index, search_size=1, + limit_to_size=True) + if response: + return response[0].get('_source').get(self.ALLOCATED_BUDGET) + return 0 + + @typeguard.typechecked + @logger_time_stamp + def get_user_cost_data(self, group_by_tag_name: str, group_by_tag_value: str, requested_date: datetime = '', forecast: bool = False, duration: int = 0, extra_filter_key_values: dict = None): + """ + This method fetches data from the es_reports + :param extra_filter_key_values: + :param group_by_tag_value: + :param group_by_tag_name: + :param duration: + :param forecast: + :param requested_date: + :return: + """ + extra_filter_matches = [{'Tags': {'Key': group_by_tag_name, 'Values': [group_by_tag_value]}}] + tags = {group_by_tag_name: group_by_tag_value} + if extra_filter_key_values: + tags.update({{filter_key: filter_value} for filter_key, filter_value in extra_filter_key_values.items()}) + start_date = requested_date.replace(minute=self.ZERO, hour=self.ZERO, second=self.ZERO, microsecond=self.ZERO) + response = {} + if forecast: + # Todo Will Add in future release + resource_type = 'Forecast' + pass + else: + end_date = datetime.utcnow().replace(microsecond=self.ZERO) + timedelta(days=1) + response = self.__cost_over_usage.get_monthly_user_es_cost_data(start_date=start_date, end_date=end_date, + extra_matches=extra_filter_matches, + extra_operation=self.AND, + tag_name=group_by_tag_name, tags=tags) + resource_type = 'Cost' + if response: + return round(response[self.ZERO].get(resource_type), self.DEFAULT_ROUND_DIGITS) + return self.ZERO + + @typeguard.typechecked + def _upload_cro_report_to_es(self, monitor_data: dict): + """ + This method uploads the data to elastic search index and return the data + :param monitor_data: + :return: + """ + upload_data = {} + for ticket_id, instance_data in monitor_data.items(): + if instance_data: + ticket_id = ticket_id.split('-')[-1] + user = instance_data[self.ZERO].get('user') + issue_description = self._jira_operations.get_issue_description(ticket_id=ticket_id, state='ANY') + ticket_opened_date = issue_description.get('TicketOpenedDate') + group_by_tag_name = self.TICKET_ID_VALUE + user_cost = self.get_user_cost_data(group_by_tag_name=group_by_tag_name, group_by_tag_value=ticket_id, + requested_date=ticket_opened_date) + cost_estimation = float(instance_data[self.ZERO].get('estimated_cost', self.ZERO)) + if self._es_operations.verify_elastic_index_doc_id(index=self.__cost_over_usage.es_index_cro, + doc_id=ticket_id): + if self._check_value_in_es(tag_key='ticket_id_state', tag_value='in-progress', ticket_id=ticket_id): + es_data = self._es_operations.get_es_data_by_id(id=ticket_id, index=self.__cost_over_usage.es_index_cro) + es_data['_source']['ticket_opened_date'] = ticket_opened_date.date() + es_data['_source']['user'] = user + source = self._prepare_update_es_data(source=es_data.get('_source'), instance_data=instance_data, cost_estimation=cost_estimation, user_cost=user_cost) + self._es_operations.update_elasticsearch_index(index=self._es_index_cro, id=ticket_id, metadata=source) + upload_data[ticket_id] = source + else: + if ticket_id not in upload_data: + source = self._prepare_instance_data(instance_data=instance_data, ticket_id=ticket_id, + cost_estimation=cost_estimation, user=user, + user_cost=user_cost, ticket_opened_date=ticket_opened_date) + source['ticket_opened_date'] = ticket_opened_date.date() + source['user'] = user + if not source.get(self.ALLOCATED_BUDGET): + source[self.ALLOCATED_BUDGET] = self._get_account_budget_from_payer_ce_report() + self.__cost_over_usage.es_operations.upload_to_elasticsearch(index=self._es_index_cro, data=source, id=ticket_id) + upload_data[ticket_id] = source + return upload_data + + def _get_total_account_usage_cost(self): + """ + This method returns the total account budget till date for this year + :return: + """ + current_date = datetime.utcnow() + start_date = datetime(current_date.year, 1, 1, 0, 0, 0) + end_date = current_date + timedelta(days=1) + cost_explorer_operations = self.__cost_over_usage.get_cost_management_object() + response = cost_explorer_operations.get_usage(scope=self.__scope, start_date=start_date, end_date=end_date, + granularity='Monthly') + total_cost = cost_explorer_operations.get_total_cost(cost_data=response) + return total_cost + + @logger_time_stamp + def update_in_progress_ticket_cost(self): + """ + This method updates the in-progress tickets costs + :return: + """ + query = {"query": {"bool": {"must": [ + {"term": {"cloud_name.keyword": self._public_cloud_name}}, + {"term": {"account_name.keyword": self._account_name.upper()}}, + {"term": {"ticket_id_state.keyword": "in-progress"}} + ] + }}} + in_progress_es_tickets = self._es_operations.fetch_data_by_es_query(query=query, es_index=self._es_index_cro) + total_account_cost = self._get_total_account_usage_cost() + for in_progress_ticket in in_progress_es_tickets: + source_data = in_progress_ticket.get('_source') + ticket_id = source_data.get(self.TICKET_ID_KEY) + if source_data.get('account_name').lower() in self._account_name.lower(): + ticket_opened_date = datetime.strptime(source_data.get('ticket_opened_date'), "%Y-%m-%d") + group_by_tag_name = self.TICKET_ID_VALUE + user_cost = self.get_user_cost_data(group_by_tag_name=group_by_tag_name, group_by_tag_value=ticket_id, + requested_date=ticket_opened_date) + user_daily_cost = eval(source_data.get('user_daily_cost', "{}")) + user_name = source_data.get('user') + if not user_name: + user_name = source_data.get('user_cro') + ce_user_daily_report = self.__get_user_daily_usage_report(days=4, group_by_tag_value=ticket_id, + group_by_tag_name=group_by_tag_name, + user_name=user_name) + user_daily_cost.update(ce_user_daily_report) + update_data = {'actual_cost': user_cost, 'timestamp': datetime.utcnow(), + f'TotalCurrentUsage-{datetime.utcnow().year}': total_account_cost, + 'user_daily_cost': str(user_daily_cost)} + if not source_data.get(self.ALLOCATED_BUDGET): + update_data[self.ALLOCATED_BUDGET] = self._get_account_budget_from_payer_ce_report() + self._es_operations.update_elasticsearch_index(index=self._es_index_cro, metadata=update_data, id=ticket_id) + + def __get_user_daily_usage_report(self, days: int, group_by_tag_name: str, group_by_tag_value: str, user_name: str): + """ + This method returns the users daily report from last X days + :param days: + :return: + """ + user_daily_usage_report = {} + self._get_user_usage_by_granularity(tag_name=group_by_tag_name, tag_value=group_by_tag_value, + days=days, result_back_data=user_daily_usage_report) + self._get_user_usage_by_granularity(tag_name='User', tag_value=user_name, + days=days, result_back_data=user_daily_usage_report) + return user_daily_usage_report + + def _get_user_usage_by_granularity(self, result_back_data: dict, tag_name: str, days: int, tag_value): + """ + This method returns the organized input of the usage_reports + :param result_back_data: + :param tag_name: + :param days: + :param tag_value: + :return: + """ + end_date = datetime.utcnow() + start_date = end_date - timedelta(days=days) + cost_explorer_object = self.__cost_over_usage.get_cost_management_object() + ce_daily_usage = cost_explorer_object.get_usage(scope=self.__scope, grouping=[tag_name], granularity='Daily', + start_date=start_date, + end_date=end_date, tags={tag_name: tag_value}) + filtered_ce_daily_usage = cost_explorer_object.get_prettify_data(cost_data=ce_daily_usage) + for daily_cost in filtered_ce_daily_usage: + start_date = daily_cost.get('UsageDate') + if start_date: + start_date = str(start_date) + start_date = f'{start_date[0:4]}-{start_date[4:6]}-{start_date[6:]}' + usage = round(float(daily_cost.get('Cost')), self.DEFAULT_ROUND_DIGITS) + result_back_data.setdefault(start_date, {}).update({daily_cost.get('TagValue'): usage}) diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/cost_over_usage.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/cost_over_usage.py index 65ef7c18..43a1bddb 100644 --- a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/cost_over_usage.py +++ b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/cost_over_usage.py @@ -16,6 +16,14 @@ def __init__(self): self._subscription_id = self._environment_variables_dict.get('AZURE_SUBSCRIPTION_ID') self.__scope = f'subscriptions/{self._subscription_id}' + def get_cost_management_object(self): + """ + This method returns the object of cost_mgmt + :return: + :rtype: + """ + return self.__cost_mgmt_operations + def _verify_active_resources(self, tag_name: str, tag_value: str) -> bool: """ This method verifies any active virtual instances in all regions by tag_name, tag_value @@ -35,7 +43,7 @@ def _verify_active_resources(self, tag_name: str, tag_value: str) -> bool: return False def _get_cost_based_on_tag(self, start_date: str, end_date: str, tag_name: str, extra_filters: any = None, - extra_operation: str = 'And', granularity: str = None, forecast: bool = False): + extra_operation: str = 'And', granularity: str = None, forecast: bool = False, **kwargs): """ This method returns the cost results based on the tag_name :param start_date: @@ -62,14 +70,14 @@ def _get_cost_based_on_tag(self, start_date: str, end_date: str, tag_name: str, results_by_time = self.__cost_mgmt_operations.get_forecast(start_date=start_date, end_date=end_date, granularity=granularity, grouping=['user'], - scope=self.__scope + scope=self.__scope, **kwargs ) else: results_by_time = self.__cost_mgmt_operations.get_usage(scope=self.__scope, start_date=start_date, - end_date=end_date, grouping=['user'], - granularity=granularity) + end_date=end_date, grouping=[tag_name], + granularity=granularity, **kwargs) - response = self.__cost_mgmt_operations.get_filter_data(cost_data=results_by_time) + response = self.__cost_mgmt_operations.get_filter_data(cost_data=results_by_time, tag_name=tag_name) return response return {} diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/monitor_cro_resources.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/monitor_cro_resources.py index e29a01ac..afbe45ee 100644 --- a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/monitor_cro_resources.py +++ b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/monitor_cro_resources.py @@ -2,6 +2,7 @@ from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.abstract_resource import \ AbstractResource +from cloud_governance.cloud_resource_orchestration.utils.common_operations import check_name_and_get_key_from_tags from cloud_governance.cloud_resource_orchestration.utils.constant_variables import DURATION, TICKET_ID @@ -13,6 +14,27 @@ class MonitorCROResources(AbstractResource): def __init__(self): super().__init__() + def __get_common_data(self, tags: dict): + """ + This method returns the common data + :param tags: + :type tags: + :return: + :rtype: + """ + return { + # 'instance_state': 'NA', # virtual_machine.instance_view - not available + 'user_cro': self._compute_client.check_tag_name(tag_name='UserCRO', tags=tags), + 'user': self._compute_client.check_tag_name(tags=tags, tag_name='User'), + 'manager': self._compute_client.check_tag_name(tag_name='Manager', tags=tags), + 'approved_manager': self._compute_client.check_tag_name(tag_name='ApprovedManager', tags=tags), + 'owner': self._compute_client.check_tag_name(tag_name='Owner', tags=tags), + 'project': self._compute_client.check_tag_name(tag_name='Project', tags=tags), + 'email': self._compute_client.check_tag_name(tag_name='Email', tags=tags), + 'duration': self._compute_client.check_tag_name(tag_name='Duration', tags=tags, cast_type='int'), + 'estimated_cost': self._compute_client.check_tag_name(tag_name='EstimatedCost', tags=tags, cast_type='float') + } + def __monitor_instances(self): """ This method monitors resources and returns the data @@ -20,33 +42,45 @@ def __monitor_instances(self): :rtype: """ monitored_ticket_ids = {} + cluster_tickets = {} virtual_machines: [VirtualMachine] = self._compute_client.get_all_instances() for virtual_machine in virtual_machines: name, tags = virtual_machine.name, virtual_machine.tags found_duration = self._compute_client.check_tag_name(tags=tags, tag_name=DURATION) - print(virtual_machine.priority) if found_duration: ticket_id = self._compute_client.check_tag_name(tags=tags, tag_name=TICKET_ID) - monitored_ticket_ids.setdefault(ticket_id, []).append({ - 'region_name': virtual_machine.location, - 'ticket_id': ticket_id, - 'instance_id': virtual_machine.vm_id, - 'instance_create_time': virtual_machine.time_created, - 'instance_state': 'NA', # virtual_machine.instance_view - not available - 'instance_type': virtual_machine.hardware_profile.vm_size, - 'instance_running_days': 0, - 'instance_plan': virtual_machine.priority, - 'user_cro': self._compute_client.check_tag_name(tags=tags, tag_name='UserCRO'), - 'user': self._compute_client.check_tag_name(tags=tags, tag_name='User'), - 'manager': self._compute_client.check_tag_name(tags=tags, tag_name='Manager'), - 'approved_manager': self._compute_client.check_tag_name(tags=tags, tag_name='ApprovedManager'), - 'owner': self._compute_client.check_tag_name(tags=tags, tag_name='Owner'), - 'project': self._compute_client.check_tag_name(tags=tags, tag_name='Project'), - 'instance_name': virtual_machine.name, - 'email': self._compute_client.check_tag_name(tags=tags, tag_name='Email'), - 'duration': found_duration, - 'estimated_cost': self._compute_client.check_tag_name(tags=tags, tag_name='EstimatedCost') - }) + cluster_key, cluster_value = check_name_and_get_key_from_tags(tags=tags, + tag_name='kubernetes.io/cluster/') + hcp_key, hcp_name = check_name_and_get_key_from_tags(tags=tags, tag_name='api.openshift.com/name') + if hcp_name: + cluster_key = hcp_name + rosa, rosa_value = check_name_and_get_key_from_tags(tags=tags, tag_name='red-hat-clustertype') + if cluster_key: + cluster_tickets.setdefault(ticket_id, {}).setdefault(cluster_key, {}).setdefault('instance_data', + []) \ + .append(f"{self._compute_client.check_tag_name(tag_name='Name', tags=tags)}: " + f"{virtual_machine.vm_id}: {virtual_machine.priority}: " + f"{virtual_machine.hardware_profile.vm_size}: {'rosa' if rosa else 'self'}") + cluster_tickets.setdefault(ticket_id, {}).setdefault(cluster_key, {}).update({ + 'region_name': virtual_machine.location, + 'ticket_id': ticket_id, + }) + else: + resource_data = { + 'region_name': virtual_machine.location, + 'ticket_id': ticket_id, + 'instance_data': f"{virtual_machine.name}: " + f"{virtual_machine.vm_id}: {virtual_machine.priority}: " + f"{virtual_machine.hardware_profile.vm_size}", + } + resource_data.update(self.__get_common_data(tags)) + monitored_ticket_ids.setdefault(ticket_id, []).append(resource_data) + + for ticket_id, cluster_data in cluster_tickets.items(): + for cluster_id, cluster_values in cluster_data.items(): + cluster_values['cluster_name'] = cluster_id.split('/')[-1] + monitored_ticket_ids.setdefault(ticket_id, []).append(cluster_values) + return monitored_ticket_ids return monitored_ticket_ids def run(self): diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py index 19cafb6b..52c829c0 100644 --- a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py +++ b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py @@ -60,6 +60,7 @@ def __tag_instances(self): This method list the instances and tag the instances which have the tag TicketId :return: """ + ticket_id_instances = {} for resource_group in self._resource_groups: name = resource_group.name resource_group_tags = resource_group.tags @@ -84,6 +85,7 @@ def __tag_instances(self): for resource in resources_list: resource_ids.append(resource.id) self.__tag_ticket_id_found_resources(resource_ids=resource_ids, ticket_id=found_tag_value) + ticket_id_instances.setdefault(found_tag_value, []).append(resource_ids) def run(self): """=- diff --git a/cloud_governance/cloud_resource_orchestration/clouds/common/abstract_collect_cro_reports.py b/cloud_governance/cloud_resource_orchestration/clouds/common/abstract_collect_cro_reports.py new file mode 100644 index 00000000..e3be48f1 --- /dev/null +++ b/cloud_governance/cloud_resource_orchestration/clouds/common/abstract_collect_cro_reports.py @@ -0,0 +1,157 @@ +import logging +from abc import ABC +from datetime import datetime, timedelta + +import typeguard + +from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.cost_over_usage import CostOverUsage +from cloud_governance.common.clouds.aws.iam.iam_operations import IAMOperations +from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations +from cloud_governance.common.jira.jira_operations import JiraOperations +from cloud_governance.common.logger.init_logger import handler +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp +from cloud_governance.main.environment_variables import environment_variables + + +class AbstractCollectCROReports(ABC): + """ + This method collects the user/instance-id data from the cost-explorer + """ + + DEFAULT_ROUND_DIGITS = 3 + ZERO = 0 + TICKET_ID_KEY = 'ticket_id' + TICKET_ID_VALUE = 'TicketId' + AND = 'And' + ALLOCATED_BUDGET = 'AllocatedBudget' + + def __init__(self): + self._environment_variables_dict = environment_variables.environment_variables_dict + self._account_name = self._environment_variables_dict.get('account', '') + self._jira_operations = JiraOperations() + self._public_cloud_name = self._environment_variables_dict.get('PUBLIC_CLOUD_NAME', '') + self._es_index_cro = self._environment_variables_dict.get('CRO_ES_INDEX', '') + self._ce_payer_index = self._environment_variables_dict.get('CE_PAYER_INDEX') + self._es_operations = ElasticSearchOperations() + + def _get_account_budget_from_payer_ce_report(self): + """ + This method returns the total account budget + :return: + :rtype: + """ + raise NotImplementedError("Account Budget Not Implemented Error") + + @typeguard.typechecked + @logger_time_stamp + def _prepare_instance_data(self, instance_data: list, user: str, ticket_id: str, user_cost: float, + cost_estimation: float, ticket_opened_date: datetime): + """ + This method returns es data to upload + :param instance_data: + :param user: + :param ticket_id: + :param user_cost: + :param cost_estimation: + :param ticket_opened_date: + :return: dict data + """ + instance_meta_data = [] + cluster_names = [] + for data in instance_data: + if type(data.get('instance_data')) is not list: + instance_meta_data.append(data.get('instance_data')) + else: + instance_meta_data.extend(data.get('instance_data')) + if data.get('cluster_name'): + cluster_names.append(data.get('cluster_name')) + return { + 'cloud_name': self._public_cloud_name.upper(), + 'account_name': self._account_name, + 'region_name': instance_data[self.ZERO].get('region_name'), + 'user': user, + 'user_cro': instance_data[self.ZERO].get('user_cro'), + 'actual_cost': user_cost, + 'ticket_id': ticket_id, + 'ticket_id_state': 'in-progress', + 'estimated_cost': cost_estimation, + 'ticket_opened_date': ticket_opened_date.date(), + 'duration': int(instance_data[self.ZERO].get('duration')), + 'approved_manager': instance_data[self.ZERO].get('approved_manager'), + 'user_manager': instance_data[self.ZERO].get('manager'), + 'project': instance_data[self.ZERO].get('project'), + 'owner': instance_data[self.ZERO].get('owner'), + self.ALLOCATED_BUDGET: self._get_account_budget_from_payer_ce_report(), + 'instance_data': instance_meta_data, + 'cluster_names': cluster_names, + } + + @typeguard.typechecked + @logger_time_stamp + def _prepare_update_es_data(self, source: dict, instance_data: list, user_cost: float, cost_estimation: float): + """ + This method updates the values of jira id data + :param source: + :param instance_data: + :param user_cost: + :param cost_estimation: + :return: dict data + """ + es_instance_data = source.get('instance_data', []) + es_cluster_names = source.get('cluster_names', []) + for instance in instance_data: + instance_meta_data = instance.get('instance_data') + if instance.get('cluster_name') and instance.get('cluster_name') not in es_cluster_names: + source.setdefault('cluster_names', []).append(instance.get('cluster_name')) + if type(instance_meta_data) is not list: + instance_meta_data = [instance.get('instance_data')] + for data in instance_meta_data: + if data not in es_instance_data: + source.setdefault('instance_data', []).append(data) + source['cluster_names'] = list(set(source.get('cluster_names', []))) + source['duration'] = int(instance_data[self.ZERO].get('duration')) + source['estimated_cost'] = round(cost_estimation, self.DEFAULT_ROUND_DIGITS) + source['actual_cost'] = user_cost + if instance_data[self.ZERO].get('user_cro') and source.get('user_cro') != instance_data[self.ZERO].get('user_cro'): + source['user_cro'] = instance_data[self.ZERO].get('user_cro') + if instance_data[self.ZERO].get('user') and source.get('user') != instance_data[self.ZERO].get('user'): + source['user'] = instance_data[self.ZERO].get('user') + source['timestamp'] = datetime.utcnow() + if source.get('ticket_id_state') != 'in-progress': + source['ticket_id_state'] = 'in-progress' + source['approved_manager'] = instance_data[self.ZERO].get('approved_manager') + source['user_manager'] = instance_data[self.ZERO].get('manager'), + source['user_manager'] = instance_data[self.ZERO].get('manager'), + source[self.ALLOCATED_BUDGET] = self._get_account_budget_from_payer_ce_report() + return source + + def _check_value_in_es(self, tag_key: str, tag_value: str, ticket_id: str): + """ + This method returns the bool on comparing the es tag_key value and tag_value + :param tag_key: + :param tag_value: + :param ticket_id: + :return: + """ + es_data = self._es_operations.get_es_data_by_id(index=self._es_index_cro, id=ticket_id) + es_tag_value = es_data.get('_source', {}).get(tag_key.lower(), '').lower() + return es_tag_value == tag_value + + def _upload_cro_report_to_es(self, monitor_data: dict): + """ + This method uploads the data to elastic search index and return the data + :param monitor_data: + :return: + """ + raise NotImplementedError("Not Implemented") + + @typeguard.typechecked + @logger_time_stamp + def run(self, monitor_data: dict): + """ + This method runs data collection methods + :param monitor_data: + :return: + """ + result = self._upload_cro_report_to_es(monitor_data=monitor_data) + return result diff --git a/cloud_governance/cloud_resource_orchestration/clouds/common/abstract_cost_over_usage.py b/cloud_governance/cloud_resource_orchestration/clouds/common/abstract_cost_over_usage.py index ca5d63b0..1bb240d7 100644 --- a/cloud_governance/cloud_resource_orchestration/clouds/common/abstract_cost_over_usage.py +++ b/cloud_governance/cloud_resource_orchestration/clouds/common/abstract_cost_over_usage.py @@ -44,9 +44,9 @@ def __init__(self): @typeguard.typechecked @logger_time_stamp - def _get_monthly_user_es_cost_data(self, tag_name: str = 'User', start_date: datetime = None, - end_date: datetime = None, extra_matches: any = None, - granularity: str = 'MONTHLY', extra_operation: str = 'And'): + def get_monthly_user_es_cost_data(self, tag_name: str = 'User', start_date: datetime = None, + end_date: datetime = None, extra_matches: any = None, + granularity: str = 'MONTHLY', extra_operation: str = 'And', **kwargs): """ This method gets the user cost from the es-data :param tag_name: by default User @@ -60,7 +60,7 @@ def _get_monthly_user_es_cost_data(self, tag_name: str = 'User', start_date: dat start_date, end_date = self.__get_start_end_dates(start_date=start_date, end_date=end_date) return self._get_cost_based_on_tag(start_date=str(start_date), end_date=str(end_date), tag_name=tag_name, granularity=granularity, extra_filters=extra_matches, - extra_operation=extra_operation) + extra_operation=extra_operation, **kwargs) def _get_forecast_cost_data(self, tag_name: str = 'User', start_date: datetime = None, end_date: datetime = None, extra_matches: any = None, granularity: str = 'MONTHLY', extra_operation: str = 'And'): @@ -168,7 +168,7 @@ def _get_cost_over_usage_users(self): :return: """ over_usage_users = [] - current_month_users = self._get_monthly_user_es_cost_data() + current_month_users = self.get_monthly_user_es_cost_data() for user in current_month_users: user_name = str(user.get('User')) user_cost = round(user.get('Cost'), DEFAULT_ROUND_DIGITS) @@ -224,5 +224,5 @@ def _verify_active_resources(self, tag_name: str, tag_value: str) -> bool: @abstractmethod def _get_cost_based_on_tag(self, start_date: str, end_date: str, tag_name: str, extra_filters: any = None, - extra_operation: str = 'And', granularity: str = None, forecast: bool = False): + extra_operation: str = 'And', granularity: str = None, forecast: bool = False, **kwargs): raise NotImplementedError diff --git a/cloud_governance/cloud_resource_orchestration/common/cro_object.py b/cloud_governance/cloud_resource_orchestration/common/cro_object.py new file mode 100644 index 00000000..ac5f04d6 --- /dev/null +++ b/cloud_governance/cloud_resource_orchestration/common/cro_object.py @@ -0,0 +1,108 @@ +import boto3 + +from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.aws_monitor_tickets import AWSMonitorTickets +from cloud_governance.cloud_resource_orchestration.utils.common_operations import string_equal_ignore_case +from cloud_governance.main.environment_variables import environment_variables + + +class CroObject: + """ + This class implements the CRO activities + """ + + def __init__(self, public_cloud_name: str): + self.__public_cloud_name = public_cloud_name + self.__environment_variables_dict = environment_variables.environment_variables_dict + self.__run_active_regions = self.__environment_variables_dict.get('RUN_ACTIVE_REGIONS') + self.__region = self.__environment_variables_dict.get('AWS_DEFAULT_REGION', '') + + def cost_over_usage(self): + """ + This method returns the cost ove rusage object + :return: + :rtype: + """ + if string_equal_ignore_case(self.__public_cloud_name, 'aws'): + from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.cost_over_usage import CostOverUsage + return CostOverUsage() + elif string_equal_ignore_case(self.__public_cloud_name, 'azure'): + from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.cost_over_usage import \ + CostOverUsage + return CostOverUsage() + + def collect_cro_reports(self): + """ + This method returns the cro reports collection object + :return: + :rtype: + """ + if string_equal_ignore_case(self.__public_cloud_name, 'aws'): + from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.collect_cro_reports import \ + CollectCROReports + return CollectCROReports() + elif string_equal_ignore_case(self.__public_cloud_name, 'azure'): + from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.collect_cro_reports import \ + CollectCROReports + return CollectCROReports() + + def monitor_tickets(self): + """ + This method returns the cro monitor tickets object + :return: + :rtype: + """ + if string_equal_ignore_case(self.__public_cloud_name, 'aws'): + return AWSMonitorTickets() + elif string_equal_ignore_case(self.__public_cloud_name, 'azure'): + from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.azure_monitor_tickets\ + import AzureMonitorTickets + return AzureMonitorTickets() + + def get_tag_cro_resources_object(self, region_name: str): + """ + This method returns the tag cro resources object + :param region_name: + :type region_name: + :return: + :rtype: + """ + if string_equal_ignore_case(self.__public_cloud_name, 'aws'): + from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.tag_cro_instances import TagCROInstances + return TagCROInstances(region_name=region_name) + elif string_equal_ignore_case(self.__public_cloud_name, 'azure'): + from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.tag_cro_resources import \ + TagCROResources + return TagCROResources() + + def get_monitor_cro_resources_object(self, region_name: str): + """ + This method returns the monitor cro resources object + :param region_name: + :type region_name: + :return: + :rtype: + """ + if string_equal_ignore_case(self.__public_cloud_name, 'aws'): + from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.monitor_cro_instances import MonitorCROInstances + return MonitorCROInstances(region_name=region_name) + elif string_equal_ignore_case(self.__public_cloud_name, 'azure'): + from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.monitor_cro_resources import \ + MonitorCROResources + return MonitorCROResources() + + def get_active_regions(self): + """ + This method returns the regions to run cro + :return: + :rtype: + """ + active_regions = [] + if string_equal_ignore_case(self.__public_cloud_name, 'aws'): + if self.__run_active_regions: + active_regions = [region.get('RegionName') for region in + boto3.client('ec2').describe_regions()['Regions']] + else: + active_regions = [self.__region] + elif string_equal_ignore_case(self.__public_cloud_name, 'azure'): + active_regions = ['all'] + return active_regions diff --git a/cloud_governance/cloud_resource_orchestration/common/run_cro.py b/cloud_governance/cloud_resource_orchestration/common/run_cro.py new file mode 100644 index 00000000..43167916 --- /dev/null +++ b/cloud_governance/cloud_resource_orchestration/common/run_cro.py @@ -0,0 +1,111 @@ +from datetime import datetime + +from cloud_governance.cloud_resource_orchestration.common.cro_object import CroObject +from cloud_governance.cloud_resource_orchestration.utils.common_operations import string_equal_ignore_case +from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp +from cloud_governance.main.environment_variables import environment_variables + + +class RunCRO: + """This class monitors cro activities""" + + PERSISTENT_RUN_DOC_ID = f'cro_run_persistence-{datetime.utcnow().date()}' + PERSISTENT_RUN_INDEX = 'cloud_resource_orchestration_persistence_run' + + def __init__(self): + self.__environment_variables_dict = environment_variables.environment_variables_dict + self.__es_operations = ElasticSearchOperations() + self.__account = self.__environment_variables_dict.get('account', '') + self.__cloud_name = self.__environment_variables_dict.get('PUBLIC_CLOUD_NAME') + self.__cro_object = CroObject(public_cloud_name=self.__cloud_name) + self.__cost_over_usage = self.__cro_object.cost_over_usage() + self.__cro_reports = self.__cro_object.collect_cro_reports() + self.__monitor_tickets = self.__cro_object.monitor_tickets() + + @logger_time_stamp + def save_current_timestamp(self): + """ + This method saves the current timestamp + Storing timestamp for not sending multiple alerts in a day, if we run any number of times + :return: + """ + if not self.__es_operations.verify_elastic_index_doc_id(index=self.PERSISTENT_RUN_INDEX, + doc_id=self.PERSISTENT_RUN_DOC_ID): + self.__es_operations.upload_to_elasticsearch(index=self.PERSISTENT_RUN_INDEX, data={ + f'last_run_{self.__account}': datetime.utcnow()}, id=self.PERSISTENT_RUN_DOC_ID) + else: + self.__es_operations.update_elasticsearch_index(index=self.PERSISTENT_RUN_INDEX, + metadata={f'last_run_{self.__account}': datetime.utcnow()}, + id=self.PERSISTENT_RUN_DOC_ID) + + @logger_time_stamp + def __send_cro_alerts(self): + """ + This method sends the cost_over_usage alert and Ticket status alerts + :return: + """ + es_data = self.__es_operations.get_es_data_by_id(index=self.PERSISTENT_RUN_INDEX, id=self.PERSISTENT_RUN_DOC_ID) + first_run = True + try: + if es_data: + source = es_data.get('_source') + last_run_time = source.get(f'last_run_{self.__account.lower()}') + if last_run_time: + last_updated_time = datetime.strptime(last_run_time, "%Y-%m-%dT%H:%M:%S.%f").date() + if last_updated_time == datetime.utcnow().date(): + first_run = False + self.__environment_variables_dict.update({'CRO_FIRST_RUN': first_run}) + if first_run: + cost_over_usage_users = self.__cost_over_usage.run() + logger.info(f'Cost Over Usage Users list: {cost_over_usage_users}') + self.__monitor_tickets.run() + self.__cro_reports.update_in_progress_ticket_cost() + except Exception as err: + logger.error(err) + self.save_current_timestamp() + + def __run_cloud_resources(self): + """ + This method runs the public cloud resources and upload results to es + :return: + :rtype: + """ + active_regions = self.__cro_object.get_active_regions() + logger.info(f"""***** Running CloudResourceOrchestration in all Active regions: {active_regions} *****""") + for active_region in active_regions: + cro_monitor = self.__cro_object.get_monitor_cro_resources_object(region_name=active_region) + cro_tagging = self.__cro_object.get_tag_cro_resources_object(region_name=active_region) + if string_equal_ignore_case(self.__cloud_name, 'aws'): + self.__environment_variables_dict.update({'AWS_DEFAULT_REGION': active_region}) + logger.info(f"""Running CloudResourceOrchestration in region: {active_region}""") + logger.info(f"""{active_region}: -> Running CRO Tagging""") + tagging_response = cro_tagging.run() + logger.info(f'Tagged instances : {tagging_response}') + logger.info(f"""{active_region}: -> Running CRO Resource data Collection""") + monitor_response = cro_monitor.run() + if monitor_response: + cro_reports = self.__cro_reports.run(monitor_response) + logger.info(f'Cloud Orchestration Resources: {cro_reports}') + + @logger_time_stamp + def __start_cro(self): + """ + This method starts the cro process methods + 1. Send alert to cost over usage users + 2. Tag the new instances + 3. monitor and upload the new instances' data + 4. Monitor the Jira ticket progressing + :return: + """ + self.__send_cro_alerts() + self.__run_cloud_resources() + + @logger_time_stamp + def run(self): + """ + This method starts the aws CRO operations + :return: + """ + self.__start_cro() diff --git a/cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py b/cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py index 09477231..08c544b6 100644 --- a/cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py +++ b/cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py @@ -1,5 +1,4 @@ -from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.run_cro import RunCRO -from cloud_governance.cloud_resource_orchestration.clouds.azure.azure_run_cro import AzureRunCro +from cloud_governance.cloud_resource_orchestration.common.run_cro import RunCRO from cloud_governance.common.jira.jira import logger from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp from cloud_governance.main.environment_variables import environment_variables @@ -21,22 +20,7 @@ def __init__(self): self.__cloud_name = self.__environment_variables_dict.get('PUBLIC_CLOUD_NAME') self.__monitor = self.__environment_variables_dict.get('MONITOR') self.__account = self.__environment_variables_dict.get('account') - self.__aws_run_cro = RunCRO() - - @logger_time_stamp - def aws_cloud_monitor(self): - """ - This method starts AWS Cloud CRO - """ - self.__aws_run_cro.run() - - def __azure_cloud_monitor(self): - """ - This method starts Azure Cloud CRO - :return: - :rtype: - """ - AzureRunCro().run() + self.__run_cro = RunCRO() @logger_time_stamp def run_cloud_monitor(self): @@ -47,10 +31,9 @@ def run_cloud_monitor(self): """ if self.__cloud_name.upper() == self.AWS: logger.info(f'CLOUD_RESOURCE_ORCHESTRATION = True, PublicCloudName = {self.__cloud_name}, Account = {self.__account}') - self.aws_cloud_monitor() elif self.__cloud_name.upper() == self.AZURE: logger.info(f'CLOUD_RESOURCE_ORCHESTRATION = True, PublicCloudName = {self.__cloud_name}') - self.__azure_cloud_monitor() + self.__run_cro.run() def run(self): """ diff --git a/cloud_governance/common/clouds/azure/compute/common_operations.py b/cloud_governance/common/clouds/azure/compute/common_operations.py index c3c26049..b3ce056b 100644 --- a/cloud_governance/common/clouds/azure/compute/common_operations.py +++ b/cloud_governance/common/clouds/azure/compute/common_operations.py @@ -30,9 +30,11 @@ def _item_paged_iterator(self, item_paged_object: ItemPaged): pass return iterator_list - def check_tag_name(self, tags: Optional[dict], tag_name: str): + def check_tag_name(self, tags: Optional[dict], tag_name: str, cast_type: str = 'str'): """ This method checks tag is present and return its value + :param cast_type: + :type cast_type: :param tags: :param tag_name: :return: @@ -40,9 +42,28 @@ def check_tag_name(self, tags: Optional[dict], tag_name: str): if tags: for key, value in tags.items(): if string_equal_ignore_case(key, tag_name): - return value + return self.__convert_cast_type(value=str(value), type_cast=cast_type) return '' + def __convert_cast_type(self, type_cast: str, value: str): + """ + This method returns the type conversion value + :param type_cast: + :type type_cast: + :param value: + :type value: + :return: + :rtype: + """ + if type_cast == 'str': + return str(value) + elif type_cast == 'int': + return int(value) + elif type_cast == 'float': + return float(value) + else: + return str(value) + def _get_resource_group_name_from_resource_id(self, resource_id: str): """ This method returns the resource_group from resource_id diff --git a/cloud_governance/common/clouds/azure/compute/resource_group_operations.py b/cloud_governance/common/clouds/azure/compute/resource_group_operations.py index f4a13cfa..f86a77b5 100644 --- a/cloud_governance/common/clouds/azure/compute/resource_group_operations.py +++ b/cloud_governance/common/clouds/azure/compute/resource_group_operations.py @@ -15,12 +15,12 @@ def __init__(self): self.__resource_client = ResourceManagementClient(self._default_creds, subscription_id=self._subscription_id) @logger_time_stamp - def get_all_resource_groups(self) -> [ResourceGroup]: + def get_all_resource_groups(self, **kwargs) -> [ResourceGroup]: """ This method returns all resource groups present in azure subscription :return: """ - resource_groups_object: ItemPaged = self.__resource_client.resource_groups.list() + resource_groups_object: ItemPaged = self.__resource_client.resource_groups.list(**kwargs) resource_groups_list = self._item_paged_iterator(item_paged_object=resource_groups_object) return resource_groups_list diff --git a/cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py b/cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py index eab5c53f..1d368f5c 100644 --- a/cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py +++ b/cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py @@ -4,7 +4,7 @@ import pytz from azure.core.exceptions import HttpResponseError -from azure.mgmt.costmanagement.models import QueryDataset, QueryAggregation, QueryTimePeriod, QueryGrouping +from azure.mgmt.costmanagement.models import QueryTimePeriod from cloud_governance.common.clouds.azure.subscriptions.azure_operations import AzureOperations from cloud_governance.common.logger.init_logger import logger @@ -27,20 +27,23 @@ def __get_query_dataset(self, grouping: list, tags: dict, granularity: str): :return: :rtype: """ - filter_tags = [] + query_dataset = {"aggregation": {"totalCost": {"name": "Cost", "function": "Sum"}}, + "granularity": granularity, + } if tags: - for key, value in tags.items(): - filter_tags.append({'name': key, "operator": "In", 'values': [value]}) - filter_grouping = [] + filter_tags = {} + if len(tags) > 2: + for key, value in tags.items(): + and_filter = {'tags': {'name': key.lower(), "operator": "In", 'values': [value.lower()]}} + filter_tags.setdefault('and', []).append(and_filter) + else: + for key, value in tags.items(): + filter_tags = {'tags': {'name': key.lower(), "operator": "In", 'values': [value.lower()]}} + query_dataset['filter'] = filter_tags if grouping: + filter_grouping = [] for group in grouping: filter_grouping.append({"name": group.lower(), "type": "TagKey"}) - query_dataset = {"aggregation": {"totalCost": {"name": "Cost", "function": "Sum"}}, - "granularity": granularity, - } - if filter_tags: - query_dataset['filter'] = {"tags": filter_tags} - if filter_grouping: query_dataset['grouping'] = filter_grouping return query_dataset @@ -58,7 +61,6 @@ def get_usage(self, scope: str, start_date: datetime = None, end_date: datetime :param kwargs: :return: """ - try: if not start_date and not end_date: end_date = datetime.datetime.now(pytz.UTC) @@ -126,19 +128,54 @@ def get_forecast(self, scope: str, start_date: datetime = '', end_date: datetime logger.error(err) return [] - def get_filter_data(self, cost_data: dict): + def get_filter_data(self, cost_data: dict, tag_name: str = 'User'): """ This method returns the cost data in dict format + :param tag_name: + :type tag_name: :param cost_data: :type cost_data: :return: :rtype: """ + output_list = self.get_prettify_data(cost_data) + users_list = {} + for item in output_list: + tag_value = item.get('TagValue') + if tag_value not in users_list: + users_list[tag_value] = {} + users_list[tag_value]['Cost'] = item.get('Cost') + else: + users_list[tag_value]['Cost'] = users_list[tag_value]['Cost'] + item.get('Cost') + users_cost = [] + for value, cost in users_list.items(): + users_cost.append({'User': value, 'Cost': cost.get('Cost')}) + return users_cost + + def get_prettify_data(self, cost_data: dict): + """ + This method returns the prettify data + :param cost_data: + :type cost_data: + :return: + :rtype: + """ + columns = cost_data.get('columns') + columns_data = [column.get('name') for column in columns] rows = cost_data.get('rows') - output_dict = {} - for row in rows: - output_dict[row[3]] = output_dict.get(row[3], 0) + row[0] - users_list = [] - for user, cost in output_dict.items(): - users_list.append({'User': user, 'Cost': cost}) - return users_list + rows_data = [dict(zip(columns_data, row)) for row in rows] + return rows_data + + def get_total_cost(self, cost_data: dict): + """ + This method returns the total cost of the data dict + :param cost_data: + :type cost_data: + :return: + :rtype: + """ + output_list = self.get_prettify_data(cost_data) + total_sum = 0 + for item in output_list: + total_sum += item.get('Cost') + return total_sum