Added the azure volume unattached

cloud_governance/common/clouds/azure/compute/common_operations.py

@@ -14,7 +14,7 @@ def __init__(self):
         self._default_creds = DefaultAzureCredential()
         self._subscription_id = self.__environment_variables_dict.get('AZURE_SUBSCRIPTION_ID')
 
-    def _item_paged_iterator(self, item_paged_object: ItemPaged):
+    def _item_paged_iterator(self, item_paged_object: ItemPaged, as_dict: bool = False):
         """
         This method iterates the paged object and return the list
         :param item_paged_object:
@@ -24,7 +24,10 @@ def _item_paged_iterator(self, item_paged_object: ItemPaged):
         try:
             page_item = item_paged_object.next()
             while page_item:
-                iterator_list.append(page_item)
+                if as_dict:
+                    iterator_list.append(page_item.as_dict())
+                else:
+                    iterator_list.append(page_item)
                 page_item = item_paged_object.next()
         except StopIteration:
             pass

cloud_governance/common/clouds/azure/compute/compute_operations.py

@@ -72,3 +72,27 @@ def stop_vm(self, resource_id: str):
         status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name,
                                                                          vm_name=vm_name)
         return status.done()
+
+    # volumes -> disks
+    def get_all_disks(self):
+        """
+        This method returns all the disks
+        :return:
+        :rtype:
+        """
+        paged_volumes = self.__compute_client.disks.list()
+        return self._item_paged_iterator(item_paged_object=paged_volumes, as_dict=True)
+
+    def delete_disk(self, resource_id: str):
+        """
+        This method deletes the disk
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        id_key_pairs = self.get_id_dict_data(resource_id)
+        resource_group_name = id_key_pairs.get('resourcegroups')
+        disk_name = id_key_pairs.get('disks')
+        status = self.__compute_client.disks.begin_delete(resource_group_name=resource_group_name, disk_name=disk_name)
+        return status.done()

cloud_governance/common/helpers/abstract_policy_operations.py

@@ -156,3 +156,43 @@ def _get_al_instances(self):
         :rtype:
         """
         raise NotImplementedError("This method not yet implemented")
+
+    @abstractmethod
+    def _get_all_volumes(self):
+        """
+        This method returns all the volumes
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method not yet implemented")
+
+    # ES Schema format
+
+    def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str,
+                       name: str,
+                       region: str, cleanup_result: str, resource_action: str, cloud_name: str,
+                       resource_state: str,
+                       resource_type: str, **kwargs):
+        current_date = datetime.utcnow().date()
+        resource_data = {
+            'ResourceId': resource_id,
+            'User': user,
+            'SkipPolicy': skip_policy,
+            'ResourceType': resource_type,
+            'ResourceState': resource_state,
+            'CleanUpDays': cleanup_days,
+            'DryRun': dry_run,
+            'Name': name,
+            'RegionName': region,
+            f'Resource{resource_action}': cleanup_result,
+            'PublicCloud': cloud_name,
+            'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}'
+        }
+        if kwargs.get('launch_time'):
+            resource_data.update({'LaunchTime': kwargs.get('launch_time')})
+        if kwargs.get('running_days'):
+            resource_data.update({'RunningDays': kwargs.get('running_days')})
+        if kwargs.get('volume_size'):
+            resource_data.update({'VolumeSize': kwargs.get('volume_size')})
+
+        return resource_data

cloud_governance/common/helpers/aws/aws_policy_operations.py

@@ -19,7 +19,6 @@ def __init__(self):
         self._s3_client = boto3.client('s3')
         self._iam_client = boto3.client('iam')
 
-
     def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
         """
         This method returns the tag value from the tags
@@ -50,7 +49,7 @@ def _delete_resource(self, resource_id: str):
                 self._s3_client.delete_bucket(Bucket=resource_id)
             elif self._policy == 'empty_roles':
                 self._iam_client.delete_role(RoleName=resource_id)
-            elif self._policy == 'ebs_unattached':
+            elif self._policy == 'unattached_volume':
                 self._ec2_client.delete_volume(VolumeId=resource_id)
             elif self._policy == 'ip_unattached':
                 self._ec2_client.release_address(AllocationId=resource_id)
@@ -125,7 +124,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
                 self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})
             elif self._policy == 'empty_roles':
                 self._iam_client.tag_role(RoleName=resource_id, Tags=tags)
-            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached',
+            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'unattached_volume',
                                   'instance_run'):
                 self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
         except Exception as err:
@@ -139,3 +138,12 @@ def _get_al_instances(self):
         """
         instances = self._ec2_operations.get_ec2_instance_list()
         return instances
+
+    def _get_all_volumes(self, **kwargs) -> list:
+        """
+        This method returns the all volumes
+        :return:
+        :rtype:
+        """
+        volumes = self._ec2_operations.get_volumes(**kwargs)
+        return volumes

cloud_governance/common/helpers/azure/azure_policy_operations.py

@@ -43,14 +43,16 @@ def _delete_resource(self, resource_id: str):
             if self._policy == 'instance_run':
                 action = "Stopped"
                 self.compute_operations.stop_vm(resource_id=resource_id)
+            elif self._policy == 'unattached_volume':
+                self.compute_operations.delete_disk(resource_id=resource_id)
             logger.info(f'{self._policy} {action}: {resource_id}')
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
 
     def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict):
         tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
         try:
-            if self._policy == 'instance_run':
+            if self._policy in ['instance_run', 'unattached_volume']:
                 self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags)
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
@@ -63,6 +65,8 @@ def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str):
         @param tag_value:
         @return:
         """
+        if not tags:
+            tags = {}
         if self._dry_run == "yes":
             tag_value = 0
         tag_value = f'{self.CURRENT_DATE}@{tag_value}'
@@ -88,4 +92,13 @@ def _get_al_instances(self):
         :return:
         :rtype:
         """
-        return self.compute_operations.get_all_instances()
+        return self.compute_operations.get_all_instances()
+
+    def _get_all_volumes(self) -> list:
+        """
+        This method returns the volumes by state
+        :return:
+        :rtype:
+        """
+        volumes = self.compute_operations.get_all_disks()
+        return volumes

cloud_governance/main/main_oerations/main_operations.py

@@ -38,7 +38,7 @@ def run(self):
         policy_runner = self.get_policy_runner()
         for policy_type, policies in policies_list.items():
             # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
-            if self._policy in policies and self._policy in "instance_run":
+            if self._policy in policies and self._policy in ["instance_run", "unattached_volume"]:
                 policy_runner.run(source=policy_type)
                 return True
         return False

cloud_governance/policy/abstract_policies/cleanup/abstract_unattached_volume.py

@@ -0,0 +1,25 @@
+from abc import ABC
+
+
+class AbstractUnattachedVolumes(ABC):
+
+    RESOURCE_ACTION = 'Delete'
+
+    def __init__(self):
+        super().__init__()
+
+    def _unattached_volume(self):
+        """
+        This method returns the unattached available volumes
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method not yet implemented")
+
+    def run(self):
+        """
+        This method starts the unattached volume operations
+        :return:
+        :rtype:
+        """
+        return self._unattached_volume()

cloud_governance/policy/abstract_policies/cleanup/abstractinstance_run.py

@@ -41,32 +41,6 @@ def _upload_instance_type_count_to_elastic_search(self):
         self.__es_upload.es_upload_data(items=es_instance_types_data, es_index=self.INSTANCE_TYPES_ES_INDEX,
                                         set_index='index_id')
 
-    def _get_es_data_schema_format(self, resource_id: str, user: str, skip_policy: str, launch_time: datetime,
-                                   instance_type: str, instance_state: str, running_days: int, cleanup_days: int,
-                                   dry_run: str, name: str, region: str, cleanup_result: str, cloud_name: str):
-        """
-        This method returns the schema of the es
-        :return:
-        :rtype:
-        """
-        current_date = datetime.utcnow().date()
-        return {
-            'ResourceId': resource_id,
-            'User': user,
-            'SkipPolicy': skip_policy,
-            'LaunchTime': launch_time,
-            'InstanceType': instance_type,
-            'InstanceState': instance_state,
-            'RunningDays': running_days,
-            'CleanUpDays': cleanup_days,
-            'DryRun': dry_run,
-            'Name': name,
-            'RegionName': region,
-            f'Resource{self.RESOURCE_ACTION}': cleanup_result,
-            'PublicCloud': cloud_name,
-            'index-id': f'{current_date}-{cloud_name.lower()}-{self.__account.lower()}-{region.lower()}-{resource_id}-{instance_state.lower()}'
-        }
-
     @abstractmethod
     def _update_instance_type_count(self):
         """

cloud_governance/policy/aws/cleanup/instance_run.py

@@ -40,18 +40,19 @@ def _instance_run(self):
                 cleanup_result = self.verify_and_delete_resource(
                     resource_id=instance.get('InstanceId'), tags=tags,
                     clean_up_days=cleanup_days)
-                resource_data = self._get_es_data_schema_format(
+                resource_data = self._get_es_schema(
                     resource_id=instance.get('InstanceId'),
                     skip_policy=self.get_skip_policy_value(tags=tags),
                     user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
                     launch_time=instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"),
-                    instance_type=instance.get('InstanceType'),
-                    instance_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped',
+                    resource_type=instance.get('InstanceType'),
+                    resource_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped',
+                    resource_action=self.RESOURCE_ACTION,
                     running_days=running_days, cleanup_days=cleanup_days,
                     dry_run=self._dry_run,
                     name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
                     region=self._region, cleanup_result=str(cleanup_result),
-                    cloud_name=self._cloud_name
+                    cloud_name=self._cloud_name,
                 )
                 if self._force_delete and self._dry_run == 'no':
                     resource_data.update({'ForceDeleted': str(self._force_delete)})

cloud_governance/policy/aws/cleanup/unattached_volume.py

@@ -0,0 +1,49 @@
+
+
+from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations
+from cloud_governance.common.utils.utils import Utils
+from cloud_governance.policy.abstract_policies.cleanup.abstract_unattached_volume import AbstractUnattachedVolumes
+
+
+class UnattachedVolume(AbstractUnattachedVolumes, AWSPolicyOperations):
+
+    def __init__(self):
+        super().__init__()
+
+    def _unattached_volume(self):
+        """
+        This method returns the list of unattached volumes
+        :return:
+        :rtype:
+        """
+        unattached_volumes = []
+        available_volumes = self._get_all_volumes()
+        for volume in available_volumes:
+            tags = volume.get('Tags', [])
+            resource_id = volume.get('VolumeId')
+            cleanup_result = False
+            if Utils.equal_ignore_case(volume.get('State'), 'available'):
+                cleanup_days = self.get_clean_up_days_count(tags=tags)
+                cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags,
+                                                                 clean_up_days=cleanup_days)
+                resource_data = self._get_es_schema(resource_id=resource_id,
+                                                    user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                    skip_policy=self.get_skip_policy_value(tags=tags),
+                                                    cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                    name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
+                                                    region=self._region,
+                                                    cleanup_result=str(cleanup_result),
+                                                    resource_action=self.RESOURCE_ACTION,
+                                                    cloud_name=self._cloud_name,
+                                                    resource_type=volume.get('VolumeType', ''),
+                                                    resource_state=volume.get('State') if not cleanup_result else "Deleted",
+                                                    volume_size=f"{volume.get('Size')} GB"
+                                                    )
+                unattached_volumes.append(resource_data)
+            else:
+                cleanup_days = 0
+            if not cleanup_result:
+                self.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags)
+
+        return unattached_volumes
+