Added the azure volume unattached (#715)

cloud_governance/common/clouds/azure/compute/common_operations.py

@@ -14,7 +14,7 @@ def __init__(self):
         self._default_creds = DefaultAzureCredential()
         self._subscription_id = self.__environment_variables_dict.get('AZURE_SUBSCRIPTION_ID')
 
-    def _item_paged_iterator(self, item_paged_object: ItemPaged):
+    def _item_paged_iterator(self, item_paged_object: ItemPaged, as_dict: bool = False):
         """
         This method iterates the paged object and return the list
         :param item_paged_object:
@@ -24,7 +24,10 @@ def _item_paged_iterator(self, item_paged_object: ItemPaged):
         try:
             page_item = item_paged_object.next()
             while page_item:
-                iterator_list.append(page_item)
+                if as_dict:
+                    iterator_list.append(page_item.as_dict())
+                else:
+                    iterator_list.append(page_item)
                 page_item = item_paged_object.next()
         except StopIteration:
             pass

cloud_governance/common/clouds/azure/compute/compute_operations.py

@@ -72,3 +72,27 @@ def stop_vm(self, resource_id: str):
         status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name,
                                                                          vm_name=vm_name)
         return status.done()
+
+    # volumes -> disks
+    def get_all_disks(self):
+        """
+        This method returns all the disks
+        :return:
+        :rtype:
+        """
+        paged_volumes = self.__compute_client.disks.list()
+        return self._item_paged_iterator(item_paged_object=paged_volumes, as_dict=True)
+
+    def delete_disk(self, resource_id: str):
+        """
+        This method deletes the disk
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        id_key_pairs = self.get_id_dict_data(resource_id)
+        resource_group_name = id_key_pairs.get('resourcegroups')
+        disk_name = id_key_pairs.get('disks')
+        status = self.__compute_client.disks.begin_delete(resource_group_name=resource_group_name, disk_name=disk_name)
+        return status.done()

cloud_governance/main/main_oerations/main_operations.py

@@ -38,7 +38,7 @@ def run(self):
         policy_runner = self.get_policy_runner()
         for policy_type, policies in policies_list.items():
             # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
-            if self._policy in policies and self._policy in "instance_run":
+            if self._policy in policies and self._policy in ["instance_run", "unattached_volume"]:
                 policy_runner.run(source=policy_type)
                 return True
         return False

cloud_governance/policy/aws/cleanup/unattached_volume.py

@@ -0,0 +1,50 @@
+
+
+from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations
+from cloud_governance.common.utils.utils import Utils
+
+
+class UnattachedVolume(AWSPolicyOperations):
+
+    RESOURCE_ACTION = "Delete"
+
+    def __init__(self):
+        super().__init__()
+
+    def run_policy_operations(self):
+        """
+        This method returns the list of unattached volumes
+        :return:
+        :rtype:
+        """
+        unattached_volumes = []
+        available_volumes = self._get_all_volumes()
+        for volume in available_volumes:
+            tags = volume.get('Tags', [])
+            resource_id = volume.get('VolumeId')
+            cleanup_result = False
+            if Utils.equal_ignore_case(volume.get('State'), 'available'):
+                cleanup_days = self.get_clean_up_days_count(tags=tags)
+                cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags,
+                                                                 clean_up_days=cleanup_days)
+                resource_data = self._get_es_schema(resource_id=resource_id,
+                                                    user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                    skip_policy=self.get_skip_policy_value(tags=tags),
+                                                    cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                    name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
+                                                    region=self._region,
+                                                    cleanup_result=str(cleanup_result),
+                                                    resource_action=self.RESOURCE_ACTION,
+                                                    cloud_name=self._cloud_name,
+                                                    resource_type=volume.get('VolumeType', ''),
+                                                    resource_state=volume.get('State') if not cleanup_result else "Deleted",
+                                                    volume_size=f"{volume.get('Size')} GB"
+                                                    )
+                unattached_volumes.append(resource_data)
+            else:
+                cleanup_days = 0
+            if not cleanup_result:
+                self.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags)
+
+        return unattached_volumes
+

cloud_governance/policy/aws/monthly_report.py

@@ -33,7 +33,7 @@ def policy_description(self, policy_name: str):
         policy_descriptions = {
             'ec2_stop': 'Delete the stopped instances that are stopped for more than 30 days ',
             'ec2_idle': 'stops the idle instances in the last 7 days.  ( CPU < 5%, Network < 5k )',
-            'ebs_unattached': 'Delete  unattached EBS volumes, where the unused days are calculated by the last DeattachedTime',
+            'unattached_volume': 'Delete  unattached EBS volumes.',
             'ip_unattached': 'Delete all the elastic_ips that are unused',
             'unused_nat_gateway': ' Delete all unused nat gateways',
             'zombie_snapshots': 'Delete all the snapshots which the AMI does not use',

cloud_governance/policy/azure/cleanup/unattached_volume.py

@@ -0,0 +1,49 @@
+
+from cloud_governance.policy.helpers.azure.azure_policy_operations import AzurePolicyOperations
+from cloud_governance.common.utils.utils import Utils
+
+
+class UnattachedVolume(AzurePolicyOperations):
+
+    RESOURCE_ACTION = "Delete"
+
+    def __init__(self):
+        super().__init__()
+
+    def run_policy_operations(self):
+        """
+        This method returns the list of unattached volumes
+        :return:
+        :rtype:
+        """
+        unattached_volumes = []
+        available_volumes = self._get_all_volumes()
+        for volume in available_volumes:
+            tags = volume.get('tags')
+            cleanup_result = False
+            if Utils.equal_ignore_case(volume.get('disk_state'), 'Unattached'):
+                cleanup_days = self.get_clean_up_days_count(tags=tags)
+                cleanup_result = self.verify_and_delete_resource(
+                    resource_id=volume.get('id'), tags=tags,
+                    clean_up_days=cleanup_days)
+                resource_data = self._get_es_schema(resource_id=volume.get('name'),
+                                                    user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                    skip_policy=self.get_skip_policy_value(tags=tags),
+                                                    cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                    name=volume.get('name'), region=volume.get('location'),
+                                                    cleanup_result=str(cleanup_result),
+                                                    resource_action=self.RESOURCE_ACTION,
+                                                    cloud_name=self._cloud_name,
+                                                    resource_type=f"{volume.get('sku', {}).get('tier')} "
+                                                                  f"{volume.get('sku', {}).get('name')}",
+                                                    resource_state=volume.get('disk_state') if not cleanup_result else "Deleted",
+                                                    volume_size=f"{volume.get('disk_size_gb')} GB"
+                                                    )
+                unattached_volumes.append(resource_data)
+            else:
+                cleanup_days = 0
+            if not cleanup_result:
+                self.update_resource_day_count_tag(resource_id=volume.get("id"), cleanup_days=cleanup_days, tags=tags)
+
+        return unattached_volumes
+

cloud_governance/policy/helpers/abstract_policy_operations.py

@@ -159,14 +159,22 @@ def _get_all_instances(self):
         """
         raise NotImplementedError("This method not yet implemented")
 
-    def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str,
-                       name: str, region: str, cleanup_result: str, resource_action: str, cloud_name: str,
-                       resource_state: str, resource_type: str, **kwargs):
+    @abstractmethod
+    def _get_all_volumes(self):
         """
-        This method returns the es schema data format
+        This method returns all the volumes
         :return:
         :rtype:
         """
+        raise NotImplementedError("This method not yet implemented")
+
+    # ES Schema format
+
+    def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str,
+                       name: str,
+                       region: str, cleanup_result: str, resource_action: str, cloud_name: str,
+                       resource_state: str,
+                       resource_type: str, **kwargs):
         current_date = datetime.utcnow().date()
         resource_data = {
             'ResourceId': resource_id,

cloud_governance/policy/helpers/aws/aws_policy_operations.py

@@ -49,7 +49,7 @@ def _delete_resource(self, resource_id: str):
                 self._s3_client.delete_bucket(Bucket=resource_id)
             elif self._policy == 'empty_roles':
                 self._iam_client.delete_role(RoleName=resource_id)
-            elif self._policy == 'ebs_unattached':
+            elif self._policy == 'unattached_volume':
                 self._ec2_client.delete_volume(VolumeId=resource_id)
             elif self._policy == 'ip_unattached':
                 self._ec2_client.release_address(AllocationId=resource_id)
@@ -124,7 +124,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
                 self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})
             elif self._policy == 'empty_roles':
                 self._iam_client.tag_role(RoleName=resource_id, Tags=tags)
-            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached',
+            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'unattached_volume',
                                   'instance_run'):
                 self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
         except Exception as err:
@@ -141,3 +141,12 @@ def _get_all_instances(self):
 
     def run_policy_operations(self):
         raise NotImplementedError("This method needs to be implemented")
+
+    def _get_all_volumes(self, **kwargs) -> list:
+        """
+        This method returns the all volumes
+        :return:
+        :rtype:
+        """
+        volumes = self._ec2_operations.get_volumes(**kwargs)
+        return volumes

cloud_governance/policy/helpers/azure/azure_policy_operations.py

@@ -43,14 +43,16 @@ def _delete_resource(self, resource_id: str):
             if self._policy == 'instance_run':
                 action = "Stopped"
                 self.compute_operations.stop_vm(resource_id=resource_id)
+            elif self._policy == 'unattached_volume':
+                self.compute_operations.delete_disk(resource_id=resource_id)
             logger.info(f'{self._policy} {action}: {resource_id}')
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
 
     def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict):
         tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
         try:
-            if self._policy == 'instance_run':
+            if self._policy in ['instance_run', 'unattached_volume']:
                 self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags)
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
@@ -63,6 +65,8 @@ def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str):
         @param tag_value:
         @return:
         """
+        if not tags:
+            tags = {}
         if self._dry_run == "yes":
             tag_value = 0
         tag_value = f'{self.CURRENT_DATE}@{tag_value}'
@@ -92,3 +96,12 @@ def _get_all_instances(self):
 
     def run_policy_operations(self):
         raise NotImplementedError("This method needs to be implemented")
+
+    def _get_all_volumes(self) -> list:
+        """
+        This method returns the volumes by state
+        :return:
+        :rtype:
+        """
+        volumes = self.compute_operations.get_all_disks()
+        return volumes

cloud_governance/policy/policy_runners/azure/policy_runner.py

@@ -23,6 +23,7 @@ def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: b
         if isinstance(response, str):
             logger.info(response)
         else:
+            logger.info(response)
             policy_result.extend(response)
             self._upload_elastic_search.upload(data=policy_result)
         return policy_result