Added the azure volume unattached

cloud_governance/common/clouds/azure/compute/common_operations.py

@@ -14,7 +14,7 @@ def __init__(self):
         self._default_creds = DefaultAzureCredential()
         self._subscription_id = self.__environment_variables_dict.get('AZURE_SUBSCRIPTION_ID')
 
-    def _item_paged_iterator(self, item_paged_object: ItemPaged):
+    def _item_paged_iterator(self, item_paged_object: ItemPaged, as_dict: bool = False):
         """
         This method iterates the paged object and return the list
         :param item_paged_object:
@@ -24,7 +24,10 @@ def _item_paged_iterator(self, item_paged_object: ItemPaged):
         try:
             page_item = item_paged_object.next()
             while page_item:
-                iterator_list.append(page_item)
+                if as_dict:
+                    iterator_list.append(page_item.as_dict())
+                else:
+                    iterator_list.append(page_item)
                 page_item = item_paged_object.next()
         except StopIteration:
             pass

cloud_governance/common/clouds/azure/compute/compute_operations.py

@@ -72,3 +72,27 @@ def stop_vm(self, resource_id: str):
         status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name,
                                                                          vm_name=vm_name)
         return status.done()
+
+    # volumes -> disks
+    def get_all_disks(self):
+        """
+        This method returns all the disks
+        :return:
+        :rtype:
+        """
+        paged_volumes = self.__compute_client.disks.list()
+        return self._item_paged_iterator(item_paged_object=paged_volumes, as_dict=True)
+
+    def delete_disk(self, resource_id: str):
+        """
+        This method deletes the disk
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        id_key_pairs = self.get_id_dict_data(resource_id)
+        resource_group_name = id_key_pairs.get('resourcegroups')
+        disk_name = id_key_pairs.get('disks')
+        status = self.__compute_client.disks.begin_delete(resource_group_name=resource_group_name, disk_name=disk_name)
+        return status.done()

cloud_governance/common/helpers/abstract_policy_operations.py

@@ -156,3 +156,34 @@ def _get_al_instances(self):
         :rtype:
         """
         raise NotImplementedError("This method not yet implemented")
+
+        # ES Schema format
+
+    def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str,
+                       name: str,
+                       region: str, cleanup_result: str, resource_action: str, cloud_name: str,
+                       resource_state: str,
+                       resource_type: str, **kwargs):
+        current_date = datetime.utcnow().date()
+        resource_data = {
+            'ResourceId': resource_id,
+            'User': user,
+            'SkipPolicy': skip_policy,
+            'ResourceType': resource_type,
+            'ResourceState': resource_state,
+            'CleanUpDays': cleanup_days,
+            'DryRun': dry_run,
+            'Name': name,
+            'RegionName': region,
+            f'Resource{resource_action}': cleanup_result,
+            'PublicCloud': cloud_name,
+            'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}'
+        }
+        if kwargs.get('launch_time'):
+            resource_data.update({'LaunchTime': kwargs.get('launch_time')})
+        if kwargs.get('running_days'):
+            resource_data.update({'RunningDays': kwargs.get('running_days')})
+        if kwargs.get('volume_size'):
+            resource_data.update({'VolumeSize': kwargs.get('volume_size')})
+
+        return resource_data

cloud_governance/common/helpers/azure/azure_policy_operations.py

@@ -43,14 +43,16 @@ def _delete_resource(self, resource_id: str):
             if self._policy == 'instance_run':
                 action = "Stopped"
                 self.compute_operations.stop_vm(resource_id=resource_id)
+            elif self._policy == 'volumes_unattached':
+                self.compute_operations.delete_disk(resource_id=resource_id)
             logger.info(f'{self._policy} {action}: {resource_id}')
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
 
     def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict):
         tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
         try:
-            if self._policy == 'instance_run':
+            if self._policy in ['instance_run', 'volumes_unattached']:
                 self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags)
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
@@ -63,6 +65,8 @@ def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str):
         @param tag_value:
         @return:
         """
+        if not tags:
+            tags = {}
         if self._dry_run == "yes":
             tag_value = 0
         tag_value = f'{self.CURRENT_DATE}@{tag_value}'
@@ -88,4 +92,13 @@ def _get_al_instances(self):
         :return:
         :rtype:
         """
-        return self.compute_operations.get_all_instances()
+        return self.compute_operations.get_all_instances()
+
+    def _get_all_volumes(self) -> list:
+        """
+        This method returns the volumes by state
+        :return:
+        :rtype:
+        """
+        volumes = self.compute_operations.get_all_disks()
+        return volumes

cloud_governance/main/main_oerations/main_operations.py

@@ -38,7 +38,7 @@ def run(self):
         policy_runner = self.get_policy_runner()
         for policy_type, policies in policies_list.items():
             # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
-            if self._policy in policies and self._policy in "instance_run":
+            if self._policy in policies and self._policy in ["instance_run", "volumes_unattached"]:
                 policy_runner.run(source=policy_type)
                 return True
         return False

cloud_governance/policy/abstract_policies/cleanup/abstract_volumes_unattached.py

@@ -0,0 +1,25 @@
+from abc import ABC
+
+
+class AbstractUnattachedVolumes(ABC):
+
+    RESOURCE_ACTION = 'Delete'
+
+    def __init__(self):
+        super().__init__()
+
+    def _volumes_unattached(self):
+        """
+        This method returns the unattached available volumes
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method not yet implemented")
+
+    def run(self):
+        """
+        This method starts the unattached volume operations
+        :return:
+        :rtype:
+        """
+        return self._volumes_unattached()

cloud_governance/policy/abstract_policies/cleanup/abstractinstance_run.py

@@ -41,32 +41,6 @@ def _upload_instance_type_count_to_elastic_search(self):
         self.__es_upload.es_upload_data(items=es_instance_types_data, es_index=self.INSTANCE_TYPES_ES_INDEX,
                                         set_index='index_id')
 
-    def _get_es_data_schema_format(self, resource_id: str, user: str, skip_policy: str, launch_time: datetime,
-                                   instance_type: str, instance_state: str, running_days: int, cleanup_days: int,
-                                   dry_run: str, name: str, region: str, cleanup_result: str, cloud_name: str):
-        """
-        This method returns the schema of the es
-        :return:
-        :rtype:
-        """
-        current_date = datetime.utcnow().date()
-        return {
-            'ResourceId': resource_id,
-            'User': user,
-            'SkipPolicy': skip_policy,
-            'LaunchTime': launch_time,
-            'InstanceType': instance_type,
-            'InstanceState': instance_state,
-            'RunningDays': running_days,
-            'CleanUpDays': cleanup_days,
-            'DryRun': dry_run,
-            'Name': name,
-            'RegionName': region,
-            f'Resource{self.RESOURCE_ACTION}': cleanup_result,
-            'PublicCloud': cloud_name,
-            'index-id': f'{current_date}-{cloud_name.lower()}-{self.__account.lower()}-{region.lower()}-{resource_id}-{instance_state.lower()}'
-        }
-
     @abstractmethod
     def _update_instance_type_count(self):
         """

cloud_governance/policy/aws/cleanup/instance_run.py

@@ -40,18 +40,19 @@ def _instance_run(self):
                 cleanup_result = self.verify_and_delete_resource(
                     resource_id=instance.get('InstanceId'), tags=tags,
                     clean_up_days=cleanup_days)
-                resource_data = self._get_es_data_schema_format(
+                resource_data = self._get_es_schema(
                     resource_id=instance.get('InstanceId'),
                     skip_policy=self.get_skip_policy_value(tags=tags),
                     user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
                     launch_time=instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"),
-                    instance_type=instance.get('InstanceType'),
-                    instance_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped',
+                    resource_type=instance.get('InstanceType'),
+                    resource_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped',
+                    resource_action=self.RESOURCE_ACTION,
                     running_days=running_days, cleanup_days=cleanup_days,
                     dry_run=self._dry_run,
                     name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
                     region=self._region, cleanup_result=str(cleanup_result),
-                    cloud_name=self._cloud_name
+                    cloud_name=self._cloud_name,
                 )
                 if self._force_delete and self._dry_run == 'no':
                     resource_data.update({'ForceDeleted': str(self._force_delete)})

cloud_governance/policy/azure/cleanup/instance_run.py

@@ -59,16 +59,17 @@ def _instance_run(self):
                 cleanup_days = self.get_clean_up_days_count(tags=tags)
                 cleanup_result = self.verify_and_delete_resource(resource_id=vm.id, tags=tags,
                                                                  clean_up_days=cleanup_days)
-                resource_data = self._get_es_data_schema_format(
+                resource_data = self._get_es_schema(
                     resource_id=vm.name,
                     skip_policy=self.get_skip_policy_value(tags=tags),
                     user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
                     launch_time=vm.time_created,
-                    instance_type=vm.hardware_profile.vm_size,
-                    instance_state=status if cleanup_result else 'Vm Stopped',
+                    resource_type=vm.hardware_profile.vm_size,
+                    resource_state=status if cleanup_result else 'Vm Stopped',
                     running_days=running_days, cleanup_days=cleanup_days,
                     dry_run=self._dry_run,
                     name=vm.name,
+                    resource_action=self.RESOURCE_ACTION,
                     region=vm.location, cleanup_result=str(cleanup_result),
                     cloud_name=self._cloud_name
                 )

cloud_governance/policy/azure/cleanup/volumes_unattached.py

@@ -0,0 +1,48 @@
+import json
+
+from cloud_governance.common.helpers.azure.azure_policy_operations import AzurePolicyOperations
+from cloud_governance.policy.abstract_policies.cleanup.abstract_volumes_unattached import AbstractUnattachedVolumes
+
+
+class VolumesUnattached(AbstractUnattachedVolumes, AzurePolicyOperations):
+
+    def __init__(self):
+        super().__init__()
+
+    def _volumes_unattached(self):
+        """
+        This method returns the list of unattached volumes
+        :return:
+        :rtype:
+        """
+        unattached_volumes = []
+        available_volumes = self._get_all_volumes()
+        for volume in available_volumes:
+            tags = volume.get('tags')
+            cleanup_result = False
+            if volume.get('disk_state') == 'Unattached':
+                cleanup_days = self.get_clean_up_days_count(tags=tags)
+                cleanup_result = self.verify_and_delete_resource(
+                    resource_id=volume.get('id'), tags=tags,
+                    clean_up_days=cleanup_days)
+                resource_data = self._get_es_schema(resource_id=volume.get('name'),
+                                                    user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                    skip_policy=self.get_skip_policy_value(tags=tags),
+                                                    cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                    name=volume.get('name'), region=volume.get('location'),
+                                                    cleanup_result=str(cleanup_result),
+                                                    resource_action=self.RESOURCE_ACTION,
+                                                    cloud_name=self._cloud_name,
+                                                    resource_type=f"{volume.get('sku', {}).get('tier')} "
+                                                                  f"{volume.get('sku', {}).get('name')}",
+                                                    resource_state=volume.get('disk_state') if not cleanup_result else "Deleted",
+                                                    volume_size=f"{volume.get('disk_size_gb')} GB"
+                                                    )
+                unattached_volumes.append(resource_data)
+            else:
+                cleanup_days = 0
+            if not cleanup_result:
+                self.update_resource_day_count_tag(resource_id=volume.get("id"), cleanup_days=cleanup_days, tags=tags)
+
+        return unattached_volumes
+

cloud_governance/policy/policy_runners/azure/policy_runner.py

@@ -23,6 +23,7 @@ def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: b
         if isinstance(response, str):
             logger.info(response)
         else:
+            logger.info(response)
             policy_result.extend(response)
             self._upload_elastic_search.upload(data=policy_result)
         return policy_result

tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py

@@ -30,7 +30,7 @@ def test_instance_run():
     instance_run = InstanceRun()
     running_instances_data = instance_run.run()
     assert running_instances_data[0].get('ResourceStopped') == 'False'
-    assert running_instances_data[0].get('InstanceState') == 'running'
+    assert running_instances_data[0].get('ResourceState') == 'running'
 
 
 @mock_ec2

tests/unittest/cloud_governance/policy/azure/test_instance_run.py

@@ -15,6 +15,7 @@ def test_instance_run():
     :return:
     :rtype:
     """
+    environment_variables.environment_variables_dict['dry_run'] = 'yes'
     vm1 = MockVirtualMachine(tags={'User': 'mock'})
     mock_azure = MockAzure(vms=[vm1])
     mock_virtual_machines = Mock()

tests/unittest/cloud_governance/policy/azure/test_volumes_unattached.py

@@ -0,0 +1,83 @@
+from unittest.mock import Mock, patch
+
+from azure.mgmt.compute import ComputeManagementClient
+
+from cloud_governance.main.environment_variables import environment_variables
+from cloud_governance.policy.azure.cleanup.volumes_unattached import VolumesUnattached
+from tests.unittest.mocks.azure.mock_compute import MockDisk, MockAzure
+
+
+def test_volumes_unattached_dry_run_yes_0_unattached():
+    environment_variables.environment_variables_dict['dry_run'] = 'yes'
+    mock_disk1 = MockDisk(disk_state='Attached', disk_size_gb=4)
+    mock_azure = MockAzure(disks=[mock_disk1])
+    mock_virtual_machines = Mock()
+    mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks
+    with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines):
+        volume_run = VolumesUnattached()
+        response = volume_run.run()
+        assert len(response) == 0
+
+
+def test_volumes_unattached_dry_run_yes():
+    environment_variables.environment_variables_dict['dry_run'] = 'yes'
+    mock_disk1 = MockDisk(disk_state='Unattached', disk_size_gb=4)
+    mock_azure = MockAzure(disks=[mock_disk1])
+    mock_virtual_machines = Mock()
+    mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks
+    with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines):
+        volume_run = VolumesUnattached()
+        response = volume_run.run()
+        assert len(response) > 0
+        response = response[0]
+        assert response.get('ResourceDelete') == 'False'
+        assert response.get('SkipPolicy') == 'NA'
+
+
+def test_volumes_unattached_dry_run_no():
+    environment_variables.environment_variables_dict['dry_run'] = 'no'
+    environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 1
+    mock_disk1 = MockDisk(disk_state='Unattached', disk_size_gb=4)
+    mock_azure = MockAzure(disks=[mock_disk1])
+    mock_virtual_machines = Mock()
+    mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks
+    with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines):
+        volume_run = VolumesUnattached()
+        response = volume_run.run()
+        assert len(response) > 0
+        response = response[0]
+        assert response.get('ResourceDelete') == 'True'
+        assert response.get('SkipPolicy') == 'NA'
+
+
+def test_volumes_unattached_dry_run_no_7_days_action():
+    environment_variables.environment_variables_dict['dry_run'] = 'no'
+    environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7
+    mock_disk1 = MockDisk(disk_state='Unattached', disk_size_gb=4)
+    mock_azure = MockAzure(disks=[mock_disk1])
+    mock_virtual_machines = Mock()
+    mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks
+    with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines):
+        volume_run = VolumesUnattached()
+        response = volume_run.run()
+        assert len(response) > 0
+        response = response[0]
+        assert response.get('ResourceDelete') == 'False'
+        assert response.get('SkipPolicy') == 'NA'
+
+
+def test_volumes_unattached_dry_run_no_skip():
+    tags = {'Policy': 'notdelete'}
+    environment_variables.environment_variables_dict['dry_run'] = 'no'
+    environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 1
+    mock_disk1 = MockDisk(disk_state='Unattached', disk_size_gb=4, tags=tags)
+    mock_azure = MockAzure(disks=[mock_disk1])
+    mock_virtual_machines = Mock()
+    mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks
+    with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines):
+        volume_run = VolumesUnattached()
+        response = volume_run.run()
+        assert len(response) > 0
+        response = response[0]
+        assert response.get('ResourceDelete') == 'False'
+        assert response.get('SkipPolicy') == 'NOTDELETE'