Added the azure ec2_run policy

README.md

@@ -22,7 +22,7 @@ This tool support the following policies:
 
 * Real time Openshift Cluster cost, User cost
 * [ec2_idle](cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb.
-* [ec2_run](cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2.
+* [ec2_run](cloud_governance/policy/aws/cleanup/instance_run.py): running ec2.
 * [ebs_unattached](cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status.
 * [ebs_in_use](cloud_governance/policy/aws/ebs_in_use.py): in use volumes.
 * [tag_resources](cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags

cloud_governance/common/clouds/azure/compute/compute_operations.py

@@ -31,9 +31,9 @@ def get_all_instances(self) -> [VirtualMachine]:
         instances_list: [VirtualMachine] = self._item_paged_iterator(item_paged_object=instances_paged_object)
         return instances_list
 
-    def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine:
+    def get_instance_statuses(self, resource_id: str, vm_name: str) -> dict:
         """
-        This method returns the virtual machine data by taking the id
+        This method returns the virtual machine instance status
         :param vm_name:
         :type vm_name:
         :param resource_id:
@@ -42,6 +42,33 @@ def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine:
         :rtype:
         """
         resource_group_name = self._get_resource_group_name_from_resource_id(resource_id=resource_id)
-        virtual_machine = self.__compute_client.virtual_machines.get(resource_group_name=resource_group_name,
-                                                                     vm_name=vm_name)
-        return virtual_machine
+        virtual_machine = self.__compute_client.virtual_machines.instance_view(resource_group_name=resource_group_name,
+                                                                               vm_name=vm_name)
+        return virtual_machine.as_dict()
+
+    def get_id_dict_data(self, resource_id: str):
+        """
+        This method generates the vm id dictionary
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        pairs = resource_id.split('/')[1:]
+        key_pairs = {pairs[i].lower(): pairs[i + 1] for i in range(0, len(pairs), 2)}
+        return key_pairs
+
+    def stop_vm(self, resource_id: str):
+        """
+        This method stops the vm
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        id_key_pairs = self.get_id_dict_data(resource_id)
+        resource_group_name = id_key_pairs.get('resourcegroups')
+        vm_name = id_key_pairs.get('virtualmachines')
+        status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name,
+                                                                         vm_name=vm_name)
+        return status.done()

cloud_governance/common/elasticsearch/elasticsearch_operations.py

@@ -144,6 +144,8 @@ def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc'
         # Upload data to elastic search server
         if 'account' not in map(str.lower, data.keys()):
             data['account'] = self.__account
+        if data.get('index-id'):
+            kwargs['id'] = data.get('index-id')
         try:
             if isinstance(data, dict):  # JSON Object
                 self.__es.index(index=index, doc_type=doc_type, body=data, **kwargs)
@@ -310,6 +312,8 @@ def upload_data_in_bulk(self, data_items: list, index: str, **kwargs):
             for item in bulk_items:
                 if kwargs.get('id'):
                     item['_id'] = item.get(kwargs.get('id'))
+                if item.get('index-id'):
+                    item['_id'] = item.get('index-id')
                 if not item.get('timestamp'):
                     if 'CurrentDate' in item:
                         item['timestamp'] = datetime.strptime(item.get('CurrentDate'), "%Y-%m-%d")

cloud_governance/common/helpers/aws/aws_cleanup_operations.py

@@ -1,7 +1,7 @@
-import datetime
 
 import boto3
 
+from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
 from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations
 from cloud_governance.common.logger.init_logger import logger
@@ -12,11 +12,14 @@ class AWSCleanUpOperations(AbstractCleanUpOperations):
     def __init__(self):
         super().__init__()
         self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
+        self._cloud_name = 'AWS'
         self.__s3operations = S3Operations(region_name=self._region)
         self._ec2_client = boto3.client('ec2', region_name=self._region)
+        self._ec2_operations = EC2Operations(region=self._region)
         self._s3_client = boto3.client('s3')
         self._iam_client = boto3.client('iam')
 
+
     def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
         """
         This method returns the tag value from the tags
@@ -33,23 +36,6 @@ def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
                     return tag.get('Value').strip()
         return ''
 
-    def get_clean_up_days_count(self, tags: list):
-        """
-        This method returns the cleanup days count
-        :param tags:
-        :type tags:
-        :return:
-        :rtype:
-        """
-        last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount')
-        if not last_used_day:
-            return 1
-        else:
-            date, days = last_used_day.split('@')
-            if date != str(self.CURRENT_DATE):
-                return int(days) + 1
-            return 1 if int(days) == 0 else int(days)
-
     def _delete_resource(self, resource_id: str):
         """
         This method deletes the resource by verifying the policy
@@ -72,7 +58,7 @@ def _delete_resource(self, resource_id: str):
                 self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id)
             elif self._policy == 'zombie_snapshots':
                 self._ec2_client.delete_snapshot(SnapshotId=resource_id)
-            elif self._policy == 'ec2_run':
+            elif self._policy == 'instance_run':
                 self._ec2_client.stop_instances(InstanceIds=[resource_id])
                 action = "Stopped"
             logger.info(f'{self._policy} {action}: {resource_id}')
@@ -93,9 +79,9 @@ def __remove_tag_key_aws(self, tags: list):
                 custom_tags.append(tag)
         return custom_tags
 
-    def __update_tag_value(self, tags: list, tag_name: str, tag_value: str):
+    def _update_tag_value(self, tags: list, tag_name: str, tag_value: str):
         """
-        This method updates the tag_value
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
         @param tags:
         @param tag_name:
         @param tag_value:
@@ -133,13 +119,23 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
         :return:
         :rtype:
         """
-        tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
+        tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
         try:
             if self._policy == 's3_inactive':
                 self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})
             elif self._policy == 'empty_roles':
                 self._iam_client.tag_role(RoleName=resource_id, Tags=tags)
-            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', 'ec2_run'):
+            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached',
+                                  'instance_run'):
                 self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def _get_al_instances(self):
+        """
+        This method updates the instance type count to the elasticsearch
+        :return:
+        :rtype:
+        """
+        instances = self._ec2_operations.get_ec2_instance_list()
+        return instances

cloud_governance/common/helpers/azure/azure_cleanup_operations.py

@@ -0,0 +1,91 @@
+
+from cloud_governance.common.clouds.azure.compute.compute_operations import ComputeOperations
+from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations
+from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations
+from cloud_governance.common.logger.init_logger import logger
+from cloud_governance.common.utils.utils import Utils
+
+
+class AzureCleaUpOperations(AbstractCleanUpOperations):
+
+    def __init__(self):
+        self._cloud_name = 'Azure'
+        self.compute_operations = ComputeOperations()
+        self.resource_group_operations = ResourceGroupOperations()
+        super().__init__()
+
+    def get_tag_name_from_tags(self, tags: dict, tag_name: str):
+        """
+        This method returns the tag value by the tag_name
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :return:
+        :rtype:
+        """
+        if tags:
+            for key, value in tags.items():
+                if Utils.equal_ignore_case(key, tag_name):
+                    return value
+        return ''
+
+    def _delete_resource(self, resource_id: str):
+        """
+        This method deletes the
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        action = "deleted"
+        try:
+            if self._policy == 'vm_run':
+                action = "Stopped"
+                self.compute_operations.stop_vm(resource_id=resource_id)
+            logger.info(f'{self._policy} {action}: {resource_id}')
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict):
+        tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
+        try:
+            if self._policy == 'instance_run':
+                self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags)
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str):
+        """
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
+        @param tags:
+        @param tag_name:
+        @param tag_value:
+        @return:
+        """
+        if self._dry_run == "yes":
+            tag_value = 0
+        tag_value = f'{self.CURRENT_DATE}@{tag_value}'
+        found = False
+        updated_tags = {}
+        if tags:
+            for key, value in tags.items():
+                if Utils.equal_ignore_case(key, tag_name):
+                    if value.split("@")[0] != self.CURRENT_DATE:
+                        updated_tags[key] = tag_value
+                    else:
+                        if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1:
+                            updated_tags[key] = tag_value
+                    found = True
+        tags.update(updated_tags)
+        if not found:
+            tags.update({tag_name: tag_value})
+        return tags
+
+    def _get_al_instances(self):
+        """
+        This method returns the all instances list
+        :return:
+        :rtype:
+        """
+        return self.compute_operations.get_all_instances()

cloud_governance/common/helpers/cleanup_operations.py

@@ -14,13 +14,27 @@ class AbstractCleanUpOperations(ABC):
 
     def __init__(self):
         self._environment_variables_dict = environment_variables.environment_variables_dict
+        self.account = self._environment_variables_dict.get('account')
         self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION')
         self._dry_run = self._environment_variables_dict.get('dry_run')
         self._policy = self._environment_variables_dict.get('policy')
         self._force_delete = self._environment_variables_dict.get('FORCE_DELETE')
         self._resource_id = self._environment_variables_dict.get('RESOURCE_ID')
 
-    @abstractmethod
+    def calculate_days(self, create_date: Union[datetime, str]):
+        """
+        This method returns the days
+        :param create_date:
+        :type create_date:
+        :return:
+        :rtype:
+        """
+        if isinstance(create_date, str):
+            create_date = datetime.strptime(create_date, "%Y-%M-%d")
+        today = datetime.utcnow().date()
+        days = today - create_date.date()
+        return days.days
+
     def get_clean_up_days_count(self, tags: Union[list, dict]):
         """
         This method returns the cleanup days count
@@ -29,7 +43,16 @@ def get_clean_up_days_count(self, tags: Union[list, dict]):
         :return:
         :rtype:
         """
-        raise NotImplementedError("This method is Not yet implemented")
+        if self._dry_run == 'yes':
+            return 0
+        last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount')
+        if not last_used_day:
+            return 1
+        else:
+            date, days = last_used_day.split('@')
+            if date != str(self.CURRENT_DATE):
+                return int(days) + 1
+            return 1 if int(days) == 0 else int(days)
 
     @abstractmethod
     def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str):
@@ -85,7 +108,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
         """
         raise NotImplementedError("This method is Not yet implemented")
 
-    def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int,
+    def verify_and_delete_resource(self, resource_id: str, tags: Union[list, dict], clean_up_days: int,
                                    days_to_delete_resource: int = None, **kwargs):
         """
         This method verify and delete the resource by calculating the days
@@ -109,3 +132,27 @@ def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days
                             self._delete_resource(resource_id=resource_id)
                             cleanup_resources = True
         return cleanup_resources
+
+    @abstractmethod
+    def _update_tag_value(self, tags: Union[list, dict], tag_name: str, tag_value: str):
+        """
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :param tag_value:
+        :type tag_value:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    @abstractmethod
+    def _get_al_instances(self):
+        """
+        This method returns all the instances
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method not yet implemented")

cloud_governance/common/utils/utils.py

@@ -0,0 +1,51 @@
+
+import os
+
+
+class Utils:
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_cloud_policies(cloud_name: str, file_type: str = '.py', dir_dict: bool = False,
+                           exclude_policies: list = None):
+        """
+        This method returns the policies by cloud_name
+        :return:
+        :rtype:
+        """
+        cloud_name = cloud_name.lower()
+        exclude_policies = [] if not exclude_policies else exclude_policies
+        policies_dict = {}
+        policies_list = []
+        policies_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'policy', cloud_name)
+        for (dir_path, _, filenames) in os.walk(policies_path):
+            immediate_parent = dir_path.split("/")[-1]
+            for filename in filenames:
+                if not filename.startswith('__') and filename.endswith(file_type):
+                    filename = os.path.splitext(filename)[0]
+                    if filename not in exclude_policies:
+                        if dir_dict:
+                            policies_dict.setdefault(immediate_parent, []).append(filename)
+                        else:
+                            policies_list.append(filename)
+        return policies_dict if dir_dict else policies_list
+
+    @staticmethod
+    def equal_ignore_case(str1: str, str2: str, *args):
+        """
+        This method returns boolean by comparing equal in-case sensitive all strings
+        :param str1:
+        :type str1:
+        :param str2:
+        :type str2:
+        :param args:
+        :type args:
+        :return:
+        :rtype:
+        """
+        equal = str1.lower() == str2.lower()
+        for val in args:
+            equal = str1.lower() == val.lower() and equal
+        return equal