Azure: Added the ip_unattacheed policy (#728)

cloud_governance/common/clouds/azure/compute/common_operations.py

@@ -1,9 +1,10 @@
-from typing import Optional, Union
+from typing import Optional
 
 from azure.core.paging import ItemPaged
 from azure.identity import DefaultAzureCredential
 
 from cloud_governance.cloud_resource_orchestration.utils.common_operations import string_equal_ignore_case
+from cloud_governance.common.logger.init_logger import logger
 from cloud_governance.main.environment_variables import environment_variables
 
 
@@ -67,7 +68,7 @@ def __convert_cast_type(self, type_cast: str, value: str):
         else:
             return str(value)
 
-    def _get_resource_group_name_from_resource_id(self, resource_id: str):
+    def get_resource_group_name_from_resource_id(self, resource_id: str):
         """
         This method returns the resource_group from resource_id
         :param resource_id:
@@ -78,3 +79,15 @@ def _get_resource_group_name_from_resource_id(self, resource_id: str):
         id_list = resource_id.split('/')
         key_values = {id_list[i].lower(): id_list[i+1] for i in range(0, len(id_list) - 1)}
         return key_values.get('resourcegroups').lower()
+
+    def get_id_dict_data(self, resource_id: str):
+        """
+        This method generates the vm id dictionary
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        pairs = resource_id.split('/')[1:]
+        key_pairs = {pairs[i].lower(): pairs[i + 1] for i in range(0, len(pairs), 2)}
+        return key_pairs

cloud_governance/common/clouds/azure/compute/compute_operations.py

@@ -41,23 +41,11 @@ def get_instance_statuses(self, resource_id: str, vm_name: str) -> dict:
         :return:
         :rtype:
         """
-        resource_group_name = self._get_resource_group_name_from_resource_id(resource_id=resource_id)
+        resource_group_name = self.get_resource_group_name_from_resource_id(resource_id=resource_id)
         virtual_machine = self.__compute_client.virtual_machines.instance_view(resource_group_name=resource_group_name,
                                                                                vm_name=vm_name)
         return virtual_machine.as_dict()
 
-    def get_id_dict_data(self, resource_id: str):
-        """
-        This method generates the vm id dictionary
-        :param resource_id:
-        :type resource_id:
-        :return:
-        :rtype:
-        """
-        pairs = resource_id.split('/')[1:]
-        key_pairs = {pairs[i].lower(): pairs[i + 1] for i in range(0, len(pairs), 2)}
-        return key_pairs
-
     def stop_vm(self, resource_id: str):
         """
         This method stops the vm

cloud_governance/common/clouds/azure/compute/network_operations.py

@@ -0,0 +1,72 @@
+from azure.mgmt.network import NetworkManagementClient
+
+from cloud_governance.common.clouds.azure.compute.common_operations import CommonOperations
+from cloud_governance.common.utils.utils import Utils
+
+
+class NetworkOperations(CommonOperations):
+
+    def __init__(self):
+        super().__init__()
+        self.__network_client = NetworkManagementClient(self._default_creds, subscription_id=self._subscription_id)
+
+    def __get_all_public_ip_address(self):
+        """
+        This method returns all the Public address
+        :return:
+        :rtype:
+        """
+        ips = self.__network_client.public_ip_addresses.list_all()
+        return self._item_paged_iterator(item_paged_object=ips, as_dict=True)
+
+    def get_public_ipv4_addresses(self):
+        """
+        This method returns the Public IPV4
+        :return:
+        :rtype:
+        """
+        public_addresses = []
+        for ipaddress in self.__get_all_public_ip_address():
+            if Utils.equal_ignore_case(ipaddress.get('public_ip_address_version'), 'IPv4'):
+                if Utils.equal_ignore_case(ipaddress.get('public_ip_allocation_method'), 'Static'):
+                    public_addresses.append(ipaddress)
+        return public_addresses
+
+    def __get_network_interfaces(self):
+        """
+        This method returns the network interfaces
+        :return:
+        :rtype:
+        """
+        network_interfaces = self.__network_client.network_interfaces.list_all()
+        network_interfaces = self._item_paged_iterator(item_paged_object=network_interfaces, as_dict=True)
+        return network_interfaces
+
+    def get_public_ipv4_network_interfaces(self):
+        """
+        This method returns the network interfaces have the public ip address attached
+        :return:
+        :rtype:
+        """
+        public_ipv4_network_interfaces = {}
+        network_interfaces = self.__get_network_interfaces()
+        for network_interface in network_interfaces:
+            for ip_configuration in network_interface.get('ip_configurations', []):
+                if ip_configuration.get('public_ip_address', {}):
+                    public_ipv4_address_id = ip_configuration.get('public_ip_address', {}).get('id')
+                    public_ipv4_network_interfaces.setdefault(public_ipv4_address_id, []).append(network_interface)
+        return public_ipv4_network_interfaces
+
+    # delete operations
+    def release_public_ip(self, resource_id: str):
+        """
+        This method releases the public ip
+        :return:
+        :rtype:
+        """
+        id_key_pairs = self.get_id_dict_data(resource_id)
+        resource_group_name = id_key_pairs.get('resourcegroups')
+        public_ip_address_name = id_key_pairs.get('publicipaddresses')
+        status = self.__network_client.public_ip_addresses.begin_delete(resource_group_name=resource_group_name,
+                                                                        public_ip_address_name=public_ip_address_name)
+        return status.done()

cloud_governance/main/main_oerations/main_operations.py

@@ -38,7 +38,8 @@ def run(self):
         policy_runner = self.get_policy_runner()
         for policy_type, policies in policies_list.items():
             # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
-            if self._policy in policies and self._policy in ["instance_run", "unattached_volume", "cluster_run"]:
+            if self._policy in policies and self._policy in ["instance_run", "unattached_volume", "cluster_run",
+                                                             "ip_unattached"]:
                 policy_runner.run(source=policy_type)
                 return True
         return False

cloud_governance/policy/aws/ip_unattached.py

@@ -1,52 +1,55 @@
+from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations
 
-from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy
 
-
-class IpUnattached(NonClusterZombiePolicy):
+class IpUnattached(AWSPolicyOperations):
     """
     Fetched the Unused elastic_ips( based on network interface Id) and delete it after 7 days of unused,
     alert user after 4 days of unused elastic_ip.
     """
 
+    RESOURCE_ACTION = "Delete"
+
     def __init__(self):
         super().__init__()
 
-    def run(self):
+    def run_policy_operations(self):
         """
-        This method returns zombie elastic_ip's and delete if dry_run no
-        @return:
+        This method returns the list of unattached IPV4 addresses
+        :return:
+        :rtype:
         """
         addresses = self._ec2_operations.get_elastic_ips()
-        zombie_addresses = []
+        active_cluster_ids = self._get_active_cluster_ids()
+        unattached_addresses = []
         for address in addresses:
-            ip_no_used = False
             tags = address.get('Tags', [])
-            if not self._check_cluster_tag(tags=tags) or self._get_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP'):
+            cleanup_result = False
+            ip_not_used = False
+            resource_id = address.get('AllocationId')
+            cluster_tag = self._get_cluster_tag(tags=address.get('Tags'))
+            if cluster_tag not in active_cluster_ids:
                 if not address.get('NetworkInterfaceId'):
-                    ip_no_used = True
-                    unused_days = self._get_resource_last_used_days(tags=tags)
-                    eip_cost = self.resource_pricing.get_const_prices(resource_type='eip', hours=(self.DAILY_HOURS * unused_days))
-                    delta_cost = 0
-                    if unused_days == self.DAYS_TO_NOTIFY_ADMINS:
-                        delta_cost = self.resource_pricing.get_const_prices(resource_type='eip', hours=(self.DAILY_HOURS * (unused_days - self.DAYS_TO_TRIGGER_RESOURCE_MAIL)))
-                    else:
-                        if unused_days >= self.DAYS_TO_DELETE_RESOURCE:
-                            delta_cost = self.resource_pricing.get_const_prices(resource_type='eip', hours=(self.DAILY_HOURS * (unused_days - self.DAYS_TO_NOTIFY_ADMINS)))
-                    zombie_eip = self._check_resource_and_delete(resource_name='ElasticIp',
-                                                                 resource_id='AllocationId',
-                                                                 resource_type='AllocateAddress',
-                                                                 resource=address,
-                                                                 empty_days=unused_days,
-                                                                 days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE, tags=tags,
-                                                                 extra_purse=eip_cost, delta_cost=delta_cost)
-                    if zombie_eip:
-                        zombie_addresses.append({'ResourceId': address.get('AllocationId'),
-                                                 'Name': self._get_tag_name_from_tags(tags=tags),
-                                                 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'),
-                                                 'PublicIp': address.get('PublicIp'),
-                                                 'Skip': self._get_policy_value(tags=tags),
-                                                 'Days': unused_days})
+                    cleanup_days = self.get_clean_up_days_count(tags=tags)
+                    ip_not_used = True
+                    cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags,
+                                                                     clean_up_days=cleanup_days)
+                    resource_data = self._get_es_schema(resource_id=resource_id,
+                                                        user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                        skip_policy=self.get_skip_policy_value(tags=tags),
+                                                        cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                        name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
+                                                        region=self._region,
+                                                        cleanup_result=str(cleanup_result),
+                                                        resource_action=self.RESOURCE_ACTION,
+                                                        cloud_name=self._cloud_name,
+                                                        resource_type='PublicIPv4',
+                                                        resource_state='disassociated' if not cleanup_result else "Deleted"
+                                                        )
+                    unattached_addresses.append(resource_data)
                 else:
-                    unused_days = 0
-                self._update_resource_tags(resource_id=address.get('AllocationId'), tags=tags, left_out_days=unused_days, resource_left_out=ip_no_used)
-        return zombie_addresses
+                    cleanup_days = 0
+                if not cleanup_result:
+                    if self.get_tag_name_from_tags(tags, tag_name='DaysCount') or ip_not_used:
+                        self.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags)
+
+        return unattached_addresses

cloud_governance/policy/azure/cleanup/ip_unattached.py

@@ -0,0 +1,72 @@
+
+from cloud_governance.policy.helpers.azure.azure_policy_operations import AzurePolicyOperations
+
+
+class IpUnattached(AzurePolicyOperations):
+
+        RESOURCE_ACTION = "Delete"
+
+        def __init__(self):
+            super().__init__()
+            self.__network_interfaces = self.network_operations.get_public_ipv4_network_interfaces()
+
+        def __check_ipv4_not_associated(self, ipv4_address_dict: dict):
+            """
+            This method returns bool
+            :param ipv4_address_dict:
+            :type ipv4_address_dict:
+            :return:
+            :rtype:
+            """
+            found = False
+            ip_address_id = ipv4_address_dict.get('id')
+            if not ipv4_address_dict.get('ip_configuration'):
+                found = True
+            else:
+                network_interface_id = ipv4_address_dict.get('ip_configuration', {}).get('id')
+                if ip_address_id in self.__network_interfaces:
+                    for network_interface in self.__network_interfaces.get(ip_address_id):
+                        if network_interface.get('id') in network_interface_id:
+                            if not network_interface.get('virtual_machine'):
+                                found = True
+            return found
+
+        def run_policy_operations(self, volume=None):
+            """
+            This method returns the list of unattached IPV4's
+            :return:
+            :rtype:
+            """
+            unattached_ips = []
+            active_cluster_ids = self._get_active_cluster_ids()
+            public_ipv4_address = self.network_operations.get_public_ipv4_addresses()
+            for ip_address in public_ipv4_address:
+                tags = ip_address.get('tags')
+                cleanup_result = False
+                is_disassociated_ipv4 = False
+                cluster_tag = self._get_cluster_tag(tags=tags)
+                if cluster_tag not in active_cluster_ids:
+                    is_disassociated_ipv4 = self.__check_ipv4_not_associated(ip_address)
+                    if is_disassociated_ipv4:
+                        cleanup_days = self.get_clean_up_days_count(tags=tags)
+                        cleanup_result = self.verify_and_delete_resource(resource_id=ip_address.get('id'), tags=tags,
+                                                                         clean_up_days=cleanup_days)
+                        resource_data = self._get_es_schema(resource_id=ip_address.get('name'),
+                                                            user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                            skip_policy=self.get_skip_policy_value(tags=tags),
+                                                            cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                            name=ip_address.get('name'), region=ip_address.get('location'),
+                                                            cleanup_result=str(cleanup_result),
+                                                            resource_action=self.RESOURCE_ACTION,
+                                                            cloud_name=self._cloud_name,
+                                                            resource_type="PublicIPv4 Static",
+                                                            resource_state="disassociated" if not cleanup_result else "Deleted"
+                                                            )
+                        unattached_ips.append(resource_data)
+                else:
+                    cleanup_days = 0
+                if not cleanup_result:
+                    if self.get_tag_name_from_tags(tags, tag_name='DaysCount') or is_disassociated_ipv4:
+                        self.update_resource_day_count_tag(resource_id=ip_address.get("id"),
+                                                           cleanup_days=cleanup_days, tags=tags)
+            return unattached_ips

cloud_governance/policy/helpers/abstract_policy_operations.py

@@ -192,6 +192,7 @@ def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_
             'RegionName': region,
             f'Resource{resource_action}': cleanup_result,
             'PublicCloud': cloud_name,
+            'ExpireDays': self._days_to_take_action,
             'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}'
         }
         if kwargs.get('launch_time'):

cloud_governance/policy/helpers/azure/azure_policy_operations.py

@@ -1,5 +1,6 @@
 
 from cloud_governance.common.clouds.azure.compute.compute_operations import ComputeOperations
+from cloud_governance.common.clouds.azure.compute.network_operations import NetworkOperations
 from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations
 from cloud_governance.policy.helpers.abstract_policy_operations import AbstractPolicyOperations
 from cloud_governance.common.logger.init_logger import logger
@@ -11,6 +12,7 @@ class AzurePolicyOperations(AbstractPolicyOperations):
     def __init__(self):
         self._cloud_name = 'Azure'
         self.compute_operations = ComputeOperations()
+        self.network_operations = NetworkOperations()
         self.resource_group_operations = ResourceGroupOperations()
         super().__init__()
 
@@ -45,14 +47,16 @@ def _delete_resource(self, resource_id: str):
                 self.compute_operations.stop_vm(resource_id=resource_id)
             elif self._policy == 'unattached_volume':
                 self.compute_operations.delete_disk(resource_id=resource_id)
+            elif self._policy == 'ip_unattached':
+                delete_status = self.network_operations.release_public_ip(resource_id=resource_id)
             logger.info(f'{self._policy} {action}: {resource_id}')
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
 
     def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict):
         tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
         try:
-            if self._policy in ['instance_run', 'unattached_volume']:
+            if self._policy in ['instance_run', 'unattached_volume', 'ip_unattached']:
                 self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags)
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')