Added the ec2_run policy

README.md

@@ -22,7 +22,7 @@ This tool support the following policies:
 
 * Real time Openshift Cluster cost, User cost
 * [ec2_idle](cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb.
-* [ec2_run](cloud_governance/policy/aws/ec2_run.py): running ec2.
+* [ec2_run](cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2.
 * [ebs_unattached](cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status.
 * [ebs_in_use](cloud_governance/policy/aws/ebs_in_use.py): in use volumes.
 * [tag_resources](cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags

cloud_governance/common/clouds/aws/ec2/ec2_operations.py

@@ -656,3 +656,11 @@ def describe_tags(self, **kwargs):
             ec2_service_tags = self.ec2_client.describe_tags(NextToken=ec2_service_tags.get('NextToken'), **kwargs)
             tags_list.extend(ec2_service_tags.get('Tags', []))
         return tags_list
+
+    def get_running_instance(self):
+        """
+        This method returns the EC2 running instances
+        :return:
+        :rtype:
+        """
+        return self.get_ec2_instance_list(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])

cloud_governance/common/clouds/aws/s3/s3_operations.py

@@ -242,7 +242,8 @@ def get_last_objects(self, bucket: str, logs_bucket_key: str = '', policy: str =
                 date_key = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y/%m/%d")
                 key_prefix = f'{logs_bucket_key}/{policy}/{date_key}'
             objs = self.__s3_client.list_objects_v2(Bucket=bucket, Prefix=key_prefix)['Contents']
-        except:
+        except Exception as err:
+            print(err)
             return None
         get_last_modified_key = lambda obj: int(obj['LastModified'].strftime('%s'))
         full_path = [obj['Key'] for obj in sorted(objs, key=get_last_modified_key)][-1]
@@ -294,26 +295,28 @@ def find_bucket(self, bucket_name: str):
                 return True
         return False
 
-    def __get_s3_latest_policy_file(self, policy: str):
+    def __get_s3_latest_policy_file(self, policy: str, key_prefix: str = ''):
         """
         This method return latest policy logs
         @param policy:
         @return:
         """
         return self.get_last_objects(bucket=self.__bucket,
                                      logs_bucket_key=f'{self.__logs_bucket_key}/{self.__region}',
+                                     key_prefix=key_prefix,
                                      policy=policy)
 
-    def get_last_s3_policy_content(self, policy: str = '', file_name: str = '', s3_file_path: str = None):
+    def get_last_s3_policy_content(self, policy: str = '', file_name: str = '', s3_file_path: str = None,
+                                   key_prefix: str = ''):
         """
         This method return last policy content
         @return:
         """
         with tempfile.TemporaryDirectory() as temp_local_directory:
             local_file = temp_local_directory + '/' + file_name + '.gz'
             if not s3_file_path:
-                if self.__get_s3_latest_policy_file(policy=policy):
-                    s3_file_path = self.__get_s3_latest_policy_file(policy=policy)
+                if self.__get_s3_latest_policy_file(policy=policy, key_prefix=key_prefix):
+                    s3_file_path = self.__get_s3_latest_policy_file(policy=policy, key_prefix=key_prefix)
             self.download_file(bucket=self.__bucket,
                                key=str(s3_file_path),
                                download_file=file_name + '.gz',

cloud_governance/common/helpers/aws/aws_cleanup_operations.py

@@ -0,0 +1,145 @@
+import datetime
+
+import boto3
+
+from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
+from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations
+from cloud_governance.common.logger.init_logger import logger
+
+
+class AWSCleanUpOperations(AbstractCleanUpOperations):
+
+    def __init__(self):
+        super().__init__()
+        self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
+        self.__s3operations = S3Operations(region_name=self._region)
+        self._ec2_client = boto3.client('ec2', region_name=self._region)
+        self._s3_client = boto3.client('s3')
+        self._iam_client = boto3.client('iam')
+
+    def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
+        """
+        This method returns the tag value from the tags
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :return:
+        :rtype:
+        """
+        if tags:
+            for tag in tags:
+                if tag.get('Key').strip().lower() == tag_name.lower():
+                    return tag.get('Value').strip()
+        return ''
+
+    def get_clean_up_days_count(self, tags: list):
+        """
+        This method returns the cleanup days count
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount')
+        if not last_used_day:
+            return 1
+        else:
+            date, days = last_used_day.split('@')
+            if date != str(self.CURRENT_DATE):
+                return int(days) + 1
+            return 1 if int(days) == 0 else int(days)
+
+    def _delete_resource(self, resource_id: str):
+        """
+        This method deletes the resource by verifying the policy
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        action = "deleted"
+        try:
+            if self._policy == 's3_inactive':
+                self._s3_client.delete_bucket(Bucket=resource_id)
+            elif self._policy == 'empty_roles':
+                self._iam_client.delete_role(RoleName=resource_id)
+            elif self._policy == 'ebs_unattached':
+                self._ec2_client.delete_volume(VolumeId=resource_id)
+            elif self._policy == 'ip_unattached':
+                self._ec2_client.release_address(AllocationId=resource_id)
+            elif self._policy == 'unused_nat_gateway':
+                self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id)
+            elif self._policy == 'zombie_snapshots':
+                self._ec2_client.delete_snapshot(SnapshotId=resource_id)
+            elif self._policy == 'ec2_run':
+                self._ec2_client.stop_instances(InstanceIds=[resource_id])
+                action = "Stopped"
+            logger.info(f'{self._policy} {action}: {resource_id}')
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def __remove_tag_key_aws(self, tags: list):
+        """
+        This method returns the tags that does not contain key startswith aws:
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        custom_tags = []
+        for tag in tags:
+            if not tag.get('Key').lower().startswith('aws'):
+                custom_tags.append(tag)
+        return custom_tags
+
+    def __update_tag_value(self, tags: list, tag_name: str, tag_value: str):
+        """
+        This method updates the tag_value
+        @param tags:
+        @param tag_name:
+        @param tag_value:
+        @return:
+        """
+        if self._dry_run == "yes":
+            tag_value = 0
+        tag_value = f'{self.CURRENT_DATE}@{tag_value}'
+        found = False
+        if tags:
+            for tag in tags:
+                if tag.get('Key') == tag_name:
+                    if tag.get('Value').split("@")[0] != self.CURRENT_DATE:
+                        tag['Value'] = tag_value
+                    else:
+                        if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1:
+                            tag['Value'] = tag_value
+                    found = True
+        if not found:
+            tags.append({'Key': tag_name, 'Value': tag_value})
+        tags = self.__remove_tag_key_aws(tags=tags)
+        return tags
+
+    def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list, force_tag_update: str = ''):
+        """
+        This method updates the resource tags
+        :param force_tag_update:
+        :type force_tag_update:
+        :param tags:
+        :type tags:
+        :param cleanup_days:
+        :type cleanup_days:
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
+        try:
+            if self._policy == 's3_inactive':
+                self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})
+            elif self._policy == 'empty_roles':
+                self._iam_client.tag_role(RoleName=resource_id, Tags=tags)
+            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', 'ec2_run'):
+                self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')

cloud_governance/common/helpers/cleanup_operations.py

@@ -0,0 +1,111 @@
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Union
+
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class AbstractCleanUpOperations(ABC):
+
+    DAYS_TO_NOTIFY_ADMINS = 2
+    DAYS_TO_TRIGGER_RESOURCE_MAIL = 4
+    DAILY_HOURS = 24
+    CURRENT_DATE = datetime.utcnow().date().__str__()
+
+    def __init__(self):
+        self._environment_variables_dict = environment_variables.environment_variables_dict
+        self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION')
+        self._dry_run = self._environment_variables_dict.get('dry_run')
+        self._policy = self._environment_variables_dict.get('policy')
+        self._force_delete = self._environment_variables_dict.get('FORCE_DELETE')
+        self._resource_id = self._environment_variables_dict.get('RESOURCE_ID')
+
+    @abstractmethod
+    def get_clean_up_days_count(self, tags: Union[list, dict]):
+        """
+        This method returns the cleanup days count
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    @abstractmethod
+    def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str):
+        """
+        This method returns the tag_value from the tags
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    def get_skip_policy_value(self, tags: Union[list, dict]) -> str:
+        """
+        This method returns the skip value
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Policy').strip()
+        if not policy_value:
+            policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Skip').strip()
+        if policy_value:
+            return policy_value.replace('_', '').replace('-', '').upper()
+        return 'NA'
+
+    @abstractmethod
+    def _delete_resource(self, resource_id: str):
+        """
+        This method deletes the resource
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    @abstractmethod
+    def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list):
+        """
+        This method updates the resource tags
+        :param resource_id:
+        :type resource_id:
+        :param cleanup_days:
+        :type cleanup_days:
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int,
+                                   days_to_delete_resource: int = None, **kwargs):
+        """
+        This method verify and delete the resource by calculating the days
+        :return:
+        :rtype:
+        """
+        if self._resource_id == resource_id and self._force_delete and self._dry_run == 'no':
+            self._delete_resource(resource_id=resource_id)
+            return True
+        if not days_to_delete_resource:
+            days_to_delete_resource = self._days_to_take_action
+        cleanup_resources = False
+        if clean_up_days >= self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL:
+            if clean_up_days == self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL:
+                kwargs['delta_cost'] = kwargs.get('extra_purse')
+                # @Todo, If it require add email alert. May In future will add the email alert.
+            else:
+                if clean_up_days >= days_to_delete_resource:
+                    if self._dry_run == 'no':
+                        if self.get_skip_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP'):
+                            self._delete_resource(resource_id=resource_id)
+                            cleanup_resources = True
+        return cleanup_resources

cloud_governance/common/helpers/json_datetime_encoder.py

@@ -0,0 +1,10 @@
+import json
+import datetime
+
+
+class JsonDateTimeEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
+            # Serialize datetime objects to ISO 8601 format
+            return obj.isoformat()
+        return super(JsonDateTimeEncoder, self).default(obj)

cloud_governance/main/aws_main_operations.py

@@ -0,0 +1,41 @@
+import os
+
+from cloud_governance.main.environment_variables import environment_variables
+from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner
+
+
+class AWSMainOperations:
+
+    def __init__(self):
+        self.__environment_variables_dict = environment_variables.environment_variables_dict
+        self.__policy = self.__environment_variables_dict.get('policy', '')
+        self.__policy_runner = PolicyRunner()
+
+    def __get_policies(self) -> dict:
+        """
+        This method gets the aws policies
+        :return:
+        :rtype:
+        """
+        policies = {}
+        policies_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'policy', 'aws')
+        for (dirpath, dirnames, filenames) in os.walk(policies_path):
+            immediate_parent = dirpath.split("/")[-1]
+            for filename in filenames:
+                if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')):
+                    policies.setdefault(immediate_parent, []).append(os.path.splitext(filename)[0])
+        return policies
+
+    def run(self):
+        """
+        This method run the AWS Policy operations
+        :return:
+        :rtype:
+        """
+        policies_list = self.__get_policies()
+        for policy_type, policies in policies_list.items():
+            # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
+            if self.__policy in policies and self.__policy == "ec2_run":
+                self.__policy_runner.run(source=policy_type)
+                return True
+        return False