Added the tag_azure operations to cro (#666)

cloud_governance/cloud_resource_orchestration/clouds/azure/azure_run_cro.py

@@ -0,0 +1,32 @@
+from cloud_governance.cloud_resource_orchestration.clouds.azure.resource_groups.tag_cro_resources import TagCROInstances
+
+
+class AzureRunCro:
+
+    def __init__(self):
+        pass
+
+    def __run_cloud_resources(self):
+        """
+        This method run the azure resources in specified region or all regions
+        :return:
+        """
+        TagCROInstances().run()
+
+    def __start_cro(self):
+        """
+        This method start the cro process methods
+        1. Send alert to cost over usage users
+        2. Tag the new instances
+        3. monitor and upload the new instances' data
+        4. Monitor the Jira ticket progressing
+        :return:
+        """
+        self.__run_cloud_resources()
+
+    def run(self):
+        """
+        This method start the Azure CRO operations
+        :return:
+        """
+        self.__start_cro()

cloud_governance/cloud_resource_orchestration/clouds/azure/resource_groups/tag_cro_resources.py

@@ -0,0 +1,89 @@
+from cloud_governance.cloud_resource_orchestration.utils.common_operations import get_ldap_user_data
+from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations
+from cloud_governance.common.jira.jira_operations import JiraOperations
+from cloud_governance.common.logger.init_logger import logger
+
+
+class TagCROInstances:
+
+    TICKET_ID = 'TicketId'
+    DURATION = 'Duration'
+    IN_PROGRESS = 'INPROGRESS'
+
+    def __init__(self):
+        self.__resource_group_operations = ResourceGroupOperations()
+
+    def __tag_ticket_id_found_resources(self, resource_ids: list, ticket_id: str):
+        """
+        This method tags the resources with ticket_id data
+        :param resource_ids:
+        :param ticket_id:
+        :return:
+        """
+        jira_operations = JiraOperations()
+        ticket_id = ticket_id.split('-')[-1]
+        ticket_description = jira_operations.get_issue_description(ticket_id=ticket_id, state=self.IN_PROGRESS)
+        if ticket_description:
+            duration = int(ticket_description.get('Days', 0))
+            extended_duration = int(jira_operations.get_issue_sub_tasks_duration(ticket_id=ticket_id))
+            duration += extended_duration
+            estimated_cost = float(ticket_description.get('CostEstimation'))
+            budget_extend_ticket_ids = jira_operations.get_budget_extend_tickets(ticket_id=ticket_id,
+                                                                                 ticket_state='closed')
+            extended_budget = jira_operations.get_total_extend_budget(sub_ticket_ids=budget_extend_ticket_ids)
+            estimated_cost = int(estimated_cost) + int(extended_budget)
+            manager_approved = ticket_description.get('ApprovedManager')
+            user_email = ticket_description.get('EmailAddress')
+            user = user_email.split('@')[0]
+            project = ticket_description.get('Project')
+            adding_extra_tags = {'Duration': str(duration), 'EstimatedCost': str(estimated_cost),
+                                 'ApprovedManager': manager_approved, 'Project': project,
+                                 'Email': user_email, 'UserCRO': user,
+                                 'Manager': get_ldap_user_data(user=user, tag_name='ManagerName').upper(),
+                                 'Owner': get_ldap_user_data(user=user, tag_name="FullName").upper(),
+                                 'TicketId': ticket_id
+                                 }
+            tagged_resource_ids = []
+            for resource_id in resource_ids:
+                success = self.__resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=adding_extra_tags)
+                if success:
+                    tagged_resource_ids.append(resource_id)
+            logger.info(f"Tagged the resources: {tagged_resource_ids}")
+
+    def __tag_instances(self):
+        """
+        This method list the instances and tag the instances which have the tag TicketId
+        :return:
+        """
+        resource_groups = self.__resource_group_operations.get_all_resource_groups()
+        for resource_group in resource_groups:
+            name = resource_group.name
+            resource_group_tags = resource_group.tags
+            found_tag_value = self.__resource_group_operations.check_tag_name(tags=resource_group_tags, tag_name=self.TICKET_ID)
+            found_duration = self.__resource_group_operations.check_tag_name(tags=resource_group_tags, tag_name=self.DURATION)
+            resources_list = self.__resource_group_operations.get_all_resources(resource_group_name=name)
+            apply_tags = False
+            if found_tag_value and not found_duration:
+                apply_tags = True
+            if not found_tag_value:
+                for resource in resources_list:
+                    resource_tags = resource.tags
+                    found_tag_value = self.__resource_group_operations.check_tag_name(tags=resource_tags, tag_name=self.TICKET_ID)
+                    if found_tag_value:
+                        if not found_duration:
+                            found_duration = self.__resource_group_operations.check_tag_name(tags=resource_tags,  tag_name=self.DURATION)
+                        if not found_duration:
+                            apply_tags = True
+                            break
+            if apply_tags:
+                resource_ids = [resource_group.id]
+                for resource in resources_list:
+                    resource_ids.append(resource.id)
+                self.__tag_ticket_id_found_resources(resource_ids=resource_ids, ticket_id=found_tag_value)
+
+    def run(self):
+        """=-
+        This method run the tag instance methods
+        :return:
+        """
+        return self.__tag_instances()

cloud_governance/cloud_resource_orchestration/monitor/cloud_monitor.py

@@ -1,4 +1,5 @@
 from cloud_governance.cloud_resource_orchestration.clouds.aws.ec2.run_cro import RunCRO
+from cloud_governance.cloud_resource_orchestration.clouds.azure.azure_run_cro import AzureRunCro
 from cloud_governance.common.jira.jira import logger
 from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
 from cloud_governance.main.environment_variables import environment_variables
@@ -20,24 +21,36 @@ def __init__(self):
         self.__cloud_name = self.__environment_variables_dict.get('PUBLIC_CLOUD_NAME')
         self.__monitor = self.__environment_variables_dict.get('MONITOR')
         self.__account = self.__environment_variables_dict.get('account')
-        self.__run_cro = RunCRO()
+        self.__aws_run_cro = RunCRO()
 
     @logger_time_stamp
     def aws_cloud_monitor(self):
         """
-        This method ture if the cloud name is
+        This method starts AWS Cloud CRO
         """
-        self.__run_cro.run()
+        self.__aws_run_cro.run()
+
+    def __azure_cloud_monitor(self):
+        """
+        This method starts Azure Cloud CRO
+        :return:
+        :rtype:
+        """
+        AzureRunCro().run()
 
     @logger_time_stamp
     def run_cloud_monitor(self):
         """
         This method run the public cloud monitor
+        :return:
+        :rtype:
         """
-
         if self.__cloud_name.upper() == self.AWS:
             logger.info(f'CLOUD_RESOURCE_ORCHESTRATION = True, PublicCloudName = {self.__cloud_name}, Account = {self.__account}')
             self.aws_cloud_monitor()
+        elif self.__cloud_name.upper() == self.AZURE:
+            logger.info(f'CLOUD_RESOURCE_ORCHESTRATION = True, PublicCloudName = {self.__cloud_name}')
+            self.__azure_cloud_monitor()
 
     def run(self):
         """

cloud_governance/cloud_resource_orchestration/utils/common_operations.py

@@ -1,3 +1,4 @@
+from cloud_governance.main.environment_variables import environment_variables
 
 
 def string_equal_ignore_case(value1: str, value2: str, *args) -> bool:
@@ -40,3 +41,18 @@ def get_tag_value_by_name(tags: list, tag_name: str) -> str:
         if string_equal_ignore_case(key, tag_name):
             return value
     return ''
+
+
+def get_ldap_user_data(user: str, tag_name: str):
+    """
+    This method returns the ldap user tag_name
+    :param user:
+    :param tag_name:
+    :return:
+    """
+    from cloud_governance.common.ldap.ldap_search import LdapSearch
+    ldap_search = LdapSearch(ldap_host_name=environment_variables.environment_variables_dict.get('LDAP_HOST_NAME', ''))
+    user_details = ldap_search.get_user_details(user)
+    if user_details:
+        return user_details.get(tag_name)
+    return 'NA'

cloud_governance/common/clouds/azure/compute/common_operations.py

@@ -0,0 +1,44 @@
+from typing import Optional, Union
+
+from azure.core.paging import ItemPaged
+from azure.identity import DefaultAzureCredential
+
+from cloud_governance.cloud_resource_orchestration.utils.common_operations import string_equal_ignore_case
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class CommonOperations:
+
+    def __init__(self):
+        self.__environment_variables_dict = environment_variables.environment_variables_dict
+        self._default_creds = DefaultAzureCredential()
+        self._subscription_id = self.__environment_variables_dict.get('AZURE_SUBSCRIPTION_ID')
+
+    def _item_paged_iterator(self, item_paged_object: ItemPaged):
+        """
+        This method iterates the paged object and return the list
+        :param item_paged_object:
+        :return:
+        """
+        iterator_list = []
+        try:
+            page_item = item_paged_object.next()
+            while page_item:
+                iterator_list.append(page_item)
+                page_item = item_paged_object.next()
+        except StopIteration:
+            pass
+        return iterator_list
+
+    def check_tag_name(self, tags: Optional[dict], tag_name: str):
+        """
+        This method checks tag is present and return its value
+        :param tags:
+        :param tag_name:
+        :return:
+        """
+        if tags:
+            for key, value in tags.items():
+                if string_equal_ignore_case(key, tag_name):
+                    return value
+        return ''

cloud_governance/common/clouds/azure/compute/compute_operations.py

@@ -0,0 +1,22 @@
+from azure.mgmt.compute import ComputeManagementClient
+
+from cloud_governance.common.clouds.azure.compute.common_operations import CommonOperations
+from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
+
+
+class ComputeOperations(CommonOperations):
+
+    def __init__(self):
+        super().__init__()
+        self.__compute_client = ComputeManagementClient(self._default_creds, subscription_id=self._subscription_id)
+
+    @logger_time_stamp
+    def get_instances_by_resource_group(self, resource_group_name: str):
+        """
+        This method returns all the compute resources by resource group
+        :return:
+        """
+        instances_paged_object = self.__compute_client.virtual_machines.list(resource_group_name=resource_group_name)
+        instances_list = self._item_paged_iterator(item_paged_object=instances_paged_object)
+        return instances_list
+

cloud_governance/common/clouds/azure/compute/resource_group_operations.py

@@ -0,0 +1,60 @@
+import typeguard
+from azure.core.paging import ItemPaged
+from azure.core.polling import LROPoller
+from azure.mgmt.resource import ResourceManagementClient
+from azure.mgmt.resource.resources.v2022_09_01.models import ResourceGroup, GenericResourceExpanded, TagsResource, Tags
+
+from cloud_governance.common.clouds.azure.compute.common_operations import CommonOperations
+from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
+
+
+class ResourceGroupOperations(CommonOperations):
+
+    def __init__(self):
+        super().__init__()
+        self.__resource_client = ResourceManagementClient(self._default_creds, subscription_id=self._subscription_id)
+
+    @logger_time_stamp
+    def get_all_resource_groups(self) -> [ResourceGroup]:
+        """
+        This method returns all resource groups present in azure subscription
+        :return:
+        """
+        resource_groups_object: ItemPaged = self.__resource_client.resource_groups.list()
+        resource_groups_list = self._item_paged_iterator(item_paged_object=resource_groups_object)
+        return resource_groups_list
+
+    @typeguard.typechecked
+    @logger_time_stamp
+    def get_all_resources(self, resource_group_name: str) -> [GenericResourceExpanded]:
+        """
+        This method returns all the resources in a resource_group
+        :param resource_group_name:
+        :type resource_group_name:
+        :return:
+        :rtype:
+        """
+        resources_list_object: ItemPaged = self.__resource_client.resources.list_by_resource_group(resource_group_name=resource_group_name)
+        resources_list = self._item_paged_iterator(item_paged_object=resources_list_object)
+        return resources_list
+
+    def creates_or_updates_tags(self, resource_id: str, tags: dict):
+        """
+        This method creates or updates the tag on the specific resource
+        :param tags:
+        :param resource_id:
+        :return:
+        """
+        status: LROPoller = self.__resource_client.tags.begin_create_or_update_at_scope(scope=resource_id,
+                                                                                        parameters=self.__construct_tags(tags=tags))
+        return status.done()
+
+    def __construct_tags(self, tags: dict) -> TagsResource:
+        """
+        This method constructs the tags
+        :param tags:
+        :return:
+        """
+        tag_resource = TagsResource(properties=Tags(tags=tags))
+
+        return tag_resource

cloud_governance/common/clouds/azure/subscriptions/azure_operations.py

@@ -27,7 +27,7 @@ def __init__(self):
 
     def __get_subscription_id(self):
         """
-        This methods return the subscription ID
+        This method returns the subscription ID
         @return:
         """
         subscription_list = self.__subscription_client.subscriptions.list()

cloud_governance/main/environment_variables.py

@@ -91,6 +91,7 @@ def __init__(self):
         self._environment_variables_dict['AZURE_CLIENT_ID'] = EnvironmentVariables.get_env('AZURE_CLIENT_ID', '')
         self._environment_variables_dict['AZURE_TENANT_ID'] = EnvironmentVariables.get_env('AZURE_TENANT_ID', '')
         self._environment_variables_dict['AZURE_CLIENT_SECRET'] = EnvironmentVariables.get_env('AZURE_CLIENT_SECRET', '')
+        self._environment_variables_dict['AZURE_SUBSCRIPTION_ID'] = EnvironmentVariables.get_env('AZURE_SUBSCRIPTION_ID', '')
         if self._environment_variables_dict['AZURE_CLIENT_ID'] and self._environment_variables_dict['AZURE_TENANT_ID']\
                 and self._environment_variables_dict['AZURE_CLIENT_SECRET']:
             self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AZURE'

requirements.txt

@@ -2,6 +2,9 @@ aiohttp==3.8.5
 attrs==21.4.0
 azure-identity==1.12.0
 azure-mgmt-billing==6.0.0
+azure-mgmt-resource==23.0.1
+azure-mgmt-compute==30.1.0
+azure-mgmt-network==25.0.0
 azure-mgmt-costmanagement==3.0.0
 azure-mgmt-subscription==3.1.1
 boto3==1.26.4

setup.py

@@ -68,7 +68,9 @@
         'typeguard==2.13.3',  # checking types
         'typing==3.7.4.3',
         'urllib3==1.26.7',  # required by jira
-
+        'azure-mgmt-resource==23.0.1',
+        'azure-mgmt-compute==30.1.0',
+        'azure-mgmt-network==25.0.0'
     ],
 
     setup_requires=['pytest', 'pytest-runner', 'wheel', 'coverage'],