From 787c15ae831ced8537f4c46eb7b7f32b4a35b922 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Mon, 4 Sep 2023 18:00:04 +0530 Subject: [PATCH] Added the tag_azure operations to cro --- .../clouds/azure/__init__.py | 0 .../clouds/azure/azure_run_cro.py | 32 +++++++ .../clouds/azure/resource_groups/__init__.py | 0 .../resource_groups/tag_cro_resources.py | 89 +++++++++++++++++++ .../monitor/cloud_monitor.py | 21 ++++- .../utils/common_operations.py | 16 ++++ .../common/clouds/azure/compute/__init__.py | 0 .../clouds/azure/compute/common_operations.py | 44 +++++++++ .../azure/compute/compute_operations.py | 22 +++++ .../compute/resource_group_operations.py | 60 +++++++++++++ .../azure/subscriptions/azure_operations.py | 2 +- .../main/environment_variables.py | 1 + requirements.txt | 3 + setup.py | 4 +- 14 files changed, 288 insertions(+), 6 deletions(-) create mode 100644 cloud_governance/cloud_resource_orchestration/clouds/azure/__init__.py create mode 100644 cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py create mode 100644 cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/__init__.py create mode 100644 cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py create mode 100644 cloud_governance/common/clouds/azure/compute/__init__.py create mode 100644 cloud_governance/common/clouds/azure/compute/common_operations.py create mode 100644 cloud_governance/common/clouds/azure/compute/compute_operations.py create mode 100644 cloud_governance/common/clouds/azure/compute/resource_group_operations.py diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/__init__.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py new file mode 100644 index 00000000..c08cdad7 --- /dev/null +++ b/cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py @@ -0,0 +1,32 @@ +from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.tag_cro_resources import TagCROInstances + + +class AzureRunCro: + + def __init__(self): + pass + + def __run_cloud_resources(self): + """ + This method run the azure resources in specified region or all regions + :return: + """ + TagCROInstances().run() + + def __start_cro(self): + """ + This method start the cro process methods + 1. Send alert to cost over usage users + 2. Tag the new instances + 3. monitor and upload the new instances' data + 4. Monitor the Jira ticket progressing + :return: + """ + self.__run_cloud_resources() + + def run(self): + """ + This method start the Azure CRO operations + :return: + """ + self.__start_cro() diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/__init__.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py new file mode 100644 index 00000000..29fc6af5 --- /dev/null +++ b/cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py @@ -0,0 +1,89 @@ +from cloud_governance.cloud_resource_orchestration.utils.common_operations import get_ldap_user_data +from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations +from cloud_governance.common.jira.jira_operations import JiraOperations +from cloud_governance.common.logger.init_logger import logger + + +class TagCROInstances: + + TICKET_ID = 'TicketId' + DURATION = 'Duration' + IN_PROGRESS = 'INPROGRESS' + + def __init__(self): + self.__resource_group_operations = ResourceGroupOperations() + + def __tag_ticket_id_found_resources(self, resource_ids: list, ticket_id: str): + """ + This method tags the resources with ticket_id data + :param resource_ids: + :param ticket_id: + :return: + """ + jira_operations = JiraOperations() + ticket_id = ticket_id.split('-')[-1] + ticket_description = jira_operations.get_issue_description(ticket_id=ticket_id, state=self.IN_PROGRESS) + if ticket_description: + duration = int(ticket_description.get('Days', 0)) + extended_duration = int(jira_operations.get_issue_sub_tasks_duration(ticket_id=ticket_id)) + duration += extended_duration + estimated_cost = float(ticket_description.get('CostEstimation')) + budget_extend_ticket_ids = jira_operations.get_budget_extend_tickets(ticket_id=ticket_id, + ticket_state='closed') + extended_budget = jira_operations.get_total_extend_budget(sub_ticket_ids=budget_extend_ticket_ids) + estimated_cost = int(estimated_cost) + int(extended_budget) + manager_approved = ticket_description.get('ApprovedManager') + user_email = ticket_description.get('EmailAddress') + user = user_email.split('@')[0] + project = ticket_description.get('Project') + adding_extra_tags = {'Duration': str(duration), 'EstimatedCost': str(estimated_cost), + 'ApprovedManager': manager_approved, 'Project': project, + 'Email': user_email, 'UserCRO': user, + 'Manager': get_ldap_user_data(user=user, tag_name='ManagerName').upper(), + 'Owner': get_ldap_user_data(user=user, tag_name="FullName").upper(), + 'TicketId': ticket_id + } + tagged_resource_ids = [] + for resource_id in resource_ids: + success = self.__resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=adding_extra_tags) + if success: + tagged_resource_ids.append(resource_id) + logger.info(f"Tagged the resources: {tagged_resource_ids}") + + def __tag_instances(self): + """ + This method list the instances and tag the instances which have the tag TicketId + :return: + """ + resource_groups = self.__resource_group_operations.get_all_resource_groups() + for resource_group in resource_groups: + name = resource_group.name + resource_group_tags = resource_group.tags + found_tag_value = self.__resource_group_operations.check_tag_name(tags=resource_group_tags, tag_name=self.TICKET_ID) + found_duration = self.__resource_group_operations.check_tag_name(tags=resource_group_tags, tag_name=self.DURATION) + resources_list = self.__resource_group_operations.get_all_resources(resource_group_name=name) + apply_tags = False + if found_tag_value and not found_duration: + apply_tags = True + if not found_tag_value: + for resource in resources_list: + resource_tags = resource.tags + found_tag_value = self.__resource_group_operations.check_tag_name(tags=resource_tags, tag_name=self.TICKET_ID) + if found_tag_value: + if not found_duration: + found_duration = self.__resource_group_operations.check_tag_name(tags=resource_tags, tag_name=self.DURATION) + if not found_duration: + apply_tags = True + break + if apply_tags: + resource_ids = [resource_group.id] + for resource in resources_list: + resource_ids.append(resource.id) + self.__tag_ticket_id_found_resources(resource_ids=resource_ids, ticket_id=found_tag_value) + + def run(self): + """=- + This method run the tag instance methods + :return: + """ + return self.__tag_instances() diff --git a/cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py b/cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py index 3694b9e7..09477231 100644 --- a/cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py +++ b/cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py @@ -1,4 +1,5 @@ from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.run_cro import RunCRO +from cloud_governance.cloud_resource_orchestration.clouds.azure.azure_run_cro import AzureRunCro from cloud_governance.common.jira.jira import logger from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp from cloud_governance.main.environment_variables import environment_variables @@ -20,24 +21,36 @@ def __init__(self): self.__cloud_name = self.__environment_variables_dict.get('PUBLIC_CLOUD_NAME') self.__monitor = self.__environment_variables_dict.get('MONITOR') self.__account = self.__environment_variables_dict.get('account') - self.__run_cro = RunCRO() + self.__aws_run_cro = RunCRO() @logger_time_stamp def aws_cloud_monitor(self): """ - This method ture if the cloud name is + This method starts AWS Cloud CRO """ - self.__run_cro.run() + self.__aws_run_cro.run() + + def __azure_cloud_monitor(self): + """ + This method starts Azure Cloud CRO + :return: + :rtype: + """ + AzureRunCro().run() @logger_time_stamp def run_cloud_monitor(self): """ This method run the public cloud monitor + :return: + :rtype: """ - if self.__cloud_name.upper() == self.AWS: logger.info(f'CLOUD_RESOURCE_ORCHESTRATION = True, PublicCloudName = {self.__cloud_name}, Account = {self.__account}') self.aws_cloud_monitor() + elif self.__cloud_name.upper() == self.AZURE: + logger.info(f'CLOUD_RESOURCE_ORCHESTRATION = True, PublicCloudName = {self.__cloud_name}') + self.__azure_cloud_monitor() def run(self): """ diff --git a/cloud_governance/cloud_resource_orchestration/utils/common_operations.py b/cloud_governance/cloud_resource_orchestration/utils/common_operations.py index 06d71200..02368501 100644 --- a/cloud_governance/cloud_resource_orchestration/utils/common_operations.py +++ b/cloud_governance/cloud_resource_orchestration/utils/common_operations.py @@ -1,3 +1,4 @@ +from cloud_governance.main.environment_variables import environment_variables def string_equal_ignore_case(value1: str, value2: str, *args) -> bool: @@ -40,3 +41,18 @@ def get_tag_value_by_name(tags: list, tag_name: str) -> str: if string_equal_ignore_case(key, tag_name): return value return '' + + +def get_ldap_user_data(user: str, tag_name: str): + """ + This method returns the ldap user tag_name + :param user: + :param tag_name: + :return: + """ + from cloud_governance.common.ldap.ldap_search import LdapSearch + ldap_search = LdapSearch(ldap_host_name=environment_variables.environment_variables_dict.get('LDAP_HOST_NAME', '')) + user_details = ldap_search.get_user_details(user) + if user_details: + return user_details.get(tag_name) + return 'NA' diff --git a/cloud_governance/common/clouds/azure/compute/__init__.py b/cloud_governance/common/clouds/azure/compute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/clouds/azure/compute/common_operations.py b/cloud_governance/common/clouds/azure/compute/common_operations.py new file mode 100644 index 00000000..a8b7d796 --- /dev/null +++ b/cloud_governance/common/clouds/azure/compute/common_operations.py @@ -0,0 +1,44 @@ +from typing import Optional, Union + +from azure.core.paging import ItemPaged +from azure.identity import DefaultAzureCredential + +from cloud_governance.cloud_resource_orchestration.utils.common_operations import string_equal_ignore_case +from cloud_governance.main.environment_variables import environment_variables + + +class CommonOperations: + + def __init__(self): + self.__environment_variables_dict = environment_variables.environment_variables_dict + self._default_creds = DefaultAzureCredential() + self._subscription_id = self.__environment_variables_dict.get('AZURE_SUBSCRIPTION_ID') + + def _item_paged_iterator(self, item_paged_object: ItemPaged): + """ + This method iterates the paged object and return the list + :param item_paged_object: + :return: + """ + iterator_list = [] + try: + page_item = item_paged_object.next() + while page_item: + iterator_list.append(page_item) + page_item = item_paged_object.next() + except StopIteration: + pass + return iterator_list + + def check_tag_name(self, tags: Optional[dict], tag_name: str): + """ + This method checks tag is present and return its value + :param tags: + :param tag_name: + :return: + """ + if tags: + for key, value in tags.items(): + if string_equal_ignore_case(key, tag_name): + return value + return '' diff --git a/cloud_governance/common/clouds/azure/compute/compute_operations.py b/cloud_governance/common/clouds/azure/compute/compute_operations.py new file mode 100644 index 00000000..ef13e333 --- /dev/null +++ b/cloud_governance/common/clouds/azure/compute/compute_operations.py @@ -0,0 +1,22 @@ +from azure.mgmt.compute import ComputeManagementClient + +from cloud_governance.common.clouds.azure.compute.common_operations import CommonOperations +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp + + +class ComputeOperations(CommonOperations): + + def __init__(self): + super().__init__() + self.__compute_client = ComputeManagementClient(self._default_creds, subscription_id=self._subscription_id) + + @logger_time_stamp + def get_instances_by_resource_group(self, resource_group_name: str): + """ + This method returns all the compute resources by resource group + :return: + """ + instances_paged_object = self.__compute_client.virtual_machines.list(resource_group_name=resource_group_name) + instances_list = self._item_paged_iterator(item_paged_object=instances_paged_object) + return instances_list + diff --git a/cloud_governance/common/clouds/azure/compute/resource_group_operations.py b/cloud_governance/common/clouds/azure/compute/resource_group_operations.py new file mode 100644 index 00000000..f4a13cfa --- /dev/null +++ b/cloud_governance/common/clouds/azure/compute/resource_group_operations.py @@ -0,0 +1,60 @@ +import typeguard +from azure.core.paging import ItemPaged +from azure.core.polling import LROPoller +from azure.mgmt.resource import ResourceManagementClient +from azure.mgmt.resource.resources.v2022_09_01.models import ResourceGroup, GenericResourceExpanded, TagsResource, Tags + +from cloud_governance.common.clouds.azure.compute.common_operations import CommonOperations +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp + + +class ResourceGroupOperations(CommonOperations): + + def __init__(self): + super().__init__() + self.__resource_client = ResourceManagementClient(self._default_creds, subscription_id=self._subscription_id) + + @logger_time_stamp + def get_all_resource_groups(self) -> [ResourceGroup]: + """ + This method returns all resource groups present in azure subscription + :return: + """ + resource_groups_object: ItemPaged = self.__resource_client.resource_groups.list() + resource_groups_list = self._item_paged_iterator(item_paged_object=resource_groups_object) + return resource_groups_list + + @typeguard.typechecked + @logger_time_stamp + def get_all_resources(self, resource_group_name: str) -> [GenericResourceExpanded]: + """ + This method returns all the resources in a resource_group + :param resource_group_name: + :type resource_group_name: + :return: + :rtype: + """ + resources_list_object: ItemPaged = self.__resource_client.resources.list_by_resource_group(resource_group_name=resource_group_name) + resources_list = self._item_paged_iterator(item_paged_object=resources_list_object) + return resources_list + + def creates_or_updates_tags(self, resource_id: str, tags: dict): + """ + This method creates or updates the tag on the specific resource + :param tags: + :param resource_id: + :return: + """ + status: LROPoller = self.__resource_client.tags.begin_create_or_update_at_scope(scope=resource_id, + parameters=self.__construct_tags(tags=tags)) + return status.done() + + def __construct_tags(self, tags: dict) -> TagsResource: + """ + This method constructs the tags + :param tags: + :return: + """ + tag_resource = TagsResource(properties=Tags(tags=tags)) + + return tag_resource diff --git a/cloud_governance/common/clouds/azure/subscriptions/azure_operations.py b/cloud_governance/common/clouds/azure/subscriptions/azure_operations.py index 1b0127c0..5ac47a0f 100644 --- a/cloud_governance/common/clouds/azure/subscriptions/azure_operations.py +++ b/cloud_governance/common/clouds/azure/subscriptions/azure_operations.py @@ -27,7 +27,7 @@ def __init__(self): def __get_subscription_id(self): """ - This methods return the subscription ID + This method returns the subscription ID @return: """ subscription_list = self.__subscription_client.subscriptions.list() diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index a50d8ef6..719a1e2b 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -91,6 +91,7 @@ def __init__(self): self._environment_variables_dict['AZURE_CLIENT_ID'] = EnvironmentVariables.get_env('AZURE_CLIENT_ID', '') self._environment_variables_dict['AZURE_TENANT_ID'] = EnvironmentVariables.get_env('AZURE_TENANT_ID', '') self._environment_variables_dict['AZURE_CLIENT_SECRET'] = EnvironmentVariables.get_env('AZURE_CLIENT_SECRET', '') + self._environment_variables_dict['AZURE_SUBSCRIPTION_ID'] = EnvironmentVariables.get_env('AZURE_SUBSCRIPTION_ID', '') if self._environment_variables_dict['AZURE_CLIENT_ID'] and self._environment_variables_dict['AZURE_TENANT_ID']\ and self._environment_variables_dict['AZURE_CLIENT_SECRET']: self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AZURE' diff --git a/requirements.txt b/requirements.txt index 8d370857..49e4cf82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,9 @@ aiohttp==3.8.5 attrs==21.4.0 azure-identity==1.12.0 azure-mgmt-billing==6.0.0 +azure-mgmt-resource==23.0.1 +azure-mgmt-compute==30.1.0 +azure-mgmt-network==25.0.0 azure-mgmt-costmanagement==3.0.0 azure-mgmt-subscription==3.1.1 boto3==1.26.4 diff --git a/setup.py b/setup.py index 8c1a964b..281def8e 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,9 @@ 'typeguard==2.13.3', # checking types 'typing==3.7.4.3', 'urllib3==1.26.7', # required by jira - + 'azure-mgmt-resource==23.0.1', + 'azure-mgmt-compute==30.1.0', + 'azure-mgmt-network==25.0.0' ], setup_requires=['pytest', 'pytest-runner', 'wheel', 'coverage'],