Added the ec2_run policy

README.md

@@ -22,7 +22,7 @@ This tool support the following policies:
 
 * Real time Openshift Cluster cost, User cost
 * [ec2_idle](cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb.
-* [ec2_run](cloud_governance/policy/aws/ec2_run.py): running ec2.
+* [ec2_run](cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2.
 * [ebs_unattached](cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status.
 * [ebs_in_use](cloud_governance/policy/aws/ebs_in_use.py): in use volumes.
 * [tag_resources](cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags

cloud_governance/common/clouds/aws/ec2/ec2_operations.py

@@ -656,3 +656,11 @@ def describe_tags(self, **kwargs):
             ec2_service_tags = self.ec2_client.describe_tags(NextToken=ec2_service_tags.get('NextToken'), **kwargs)
             tags_list.extend(ec2_service_tags.get('Tags', []))
         return tags_list
+
+    def get_running_instance(self):
+        """
+        This method returns the EC2 running instances
+        :return:
+        :rtype:
+        """
+        return self.get_ec2_instance_list(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])

cloud_governance/common/helpers/aws/aws_cleanup_operations.py

@@ -0,0 +1,153 @@
+import datetime
+
+import boto3
+
+from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
+from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations
+from cloud_governance.common.logger.init_logger import logger
+
+
+class AWSCleanUpOperations(AbstractCleanUpOperations):
+
+    def __init__(self):
+        super().__init__()
+        self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
+        self.__s3operations = S3Operations(region_name=self._region)
+        self._ec2_client = boto3.client('ec2', region_name=self._region)
+        self._s3_client = boto3.client('s3')
+        self._iam_client = boto3.client('iam')
+
+    def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
+        if tags:
+            for tag in tags:
+                if tag.get('Key').strip().lower() == tag_name.lower():
+                    return tag.get('Value').strip()
+        return ''
+
+    def get_clean_up_days_count(self, tags: list):
+        """
+        This method returns the cleanup days count
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        if self._dry_run == 'no':
+            last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DryRunNoDays')
+        else:
+            last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DryRunYesDays')
+        if not last_used_day:
+            return 1
+        else:
+            date, days = last_used_day.split('@')
+            if date != str(datetime.datetime.now().date()):
+                return int(days) + 1
+            return 1 if int(days) == 0 else int(days)
+
+    def get_skip_policy_value(self, tags: list) -> str:
+        """
+        This method returns the skip value
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Policy').strip()
+        if not policy_value:
+            policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Skip').strip()
+        if policy_value:
+            return policy_value.replace('_', '').replace('-', '').upper()
+        return 'NA'
+
+    def _delete_resource(self, resource_id: str):
+        """
+        This method deletes the resource by verifying the policy
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        action = "deleted"
+        try:
+            if self._policy == 's3_inactive':
+                self._s3_client.delete_bucket(Bucket=resource_id)
+            elif self._policy == 'empty_roles':
+                self._iam_client.delete_role(RoleName=resource_id)
+            elif self._policy == 'ebs_unattached':
+                self._ec2_client.delete_volume(VolumeId=resource_id)
+            elif self._policy == 'ip_unattached':
+                self._ec2_client.release_address(AllocationId=resource_id)
+            elif self._policy == 'unused_nat_gateway':
+                self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id)
+            elif self._policy == 'zombie_snapshots':
+                self._ec2_client.delete_snapshot(SnapshotId=resource_id)
+            elif self._policy == 'ec2_run':
+                self._ec2_client.stop_instances(InstanceIds=[resource_id])
+                action = "Stopped"
+            logger.info(f'{self._policy} {action}: {resource_id}')
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def __update_tag_value(self, tags: list, tag_name: str, tag_value: str):
+        """
+        This method updates the tag_value
+        @param tags:
+        @param tag_name:
+        @param tag_value:
+        @return:
+        """
+        today_date = datetime.datetime.now().date().__str__()
+        tag_value = f'{today_date}@{tag_value}'
+        found = False
+        if tags:
+            for tag in tags:
+                if tag.get('Key') == tag_name:
+                    if tag.get('Value').split("@")[0] != today_date:
+                        tag['Value'] = tag_value
+                    else:
+                        if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1:
+                            tag['Value'] = tag_value
+                    found = True
+        if not found:
+            tags.append({'Key': tag_name, 'Value': tag_value})
+        return tags
+
+    def update_resource_tags(self, resource_id: str, cleanup_days: int, tags: list, force_tag_update: str = ''):
+        """
+        This method updates the resource tags
+        :param force_tag_update:
+        :type force_tag_update:
+        :param tags:
+        :type tags:
+        :param cleanup_days:
+        :type cleanup_days:
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        if force_tag_update:
+            tags = self.__update_tag_value(tags=tags, tag_name=force_tag_update, tag_value=str(cleanup_days))
+        if self._dry_run == 'no':
+            tags = self.__update_tag_value(tags=tags, tag_name='DryRunNoDays', tag_value=str(cleanup_days))
+        else:
+            tags = self.__update_tag_value(tags=tags, tag_name='DryRunYesDays', tag_value=str(cleanup_days))
+        try:
+            if self._policy == 's3_inactive':
+                self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})
+            elif self._policy == 'empty_roles':
+                self._iam_client.tag_role(RoleName=resource_id, Tags=tags)
+            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', 'ec2_run'):
+                self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def get_force_tag_update(self):
+        """
+        This method returns the value that dry_run mode other value
+        i.e if dry_run = no return DryRunYesDays
+        else return DryRunNoDays
+        :return:
+        :rtype:
+        """
+        return "DryRunYesDays" if self._dry_run == 'no' else "DryRunNoDays"

cloud_governance/common/helpers/cleanup_operations.py

@@ -0,0 +1,98 @@
+from abc import ABC, abstractmethod
+from typing import Union
+
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class AbstractCleanUpOperations(ABC):
+
+    DAYS_TO_NOTIFY_ADMINS = 2
+    DAYS_TO_TRIGGER_RESOURCE_MAIL = 4
+    DAILY_HOURS = 24
+
+    def __init__(self):
+        self._environment_variables_dict = environment_variables.environment_variables_dict
+        self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION')
+        self._dry_run = self._environment_variables_dict.get('dry_run')
+        self._policy = self._environment_variables_dict.get('policy')
+
+    @abstractmethod
+    def get_clean_up_days_count(self, tags: Union[list, dict]):
+        """
+        This method returns the cleanup days count
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    @abstractmethod
+    def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str):
+        """
+        This method returns the tag_value from the tags
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    @abstractmethod
+    def get_skip_policy_value(self, tags: Union[list, dict]):
+        """
+        This method beautify the value
+        @param tags:
+        @return:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    @abstractmethod
+    def _delete_resource(self, resource_id: str):
+        """
+        This method deletes the resource
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    @abstractmethod
+    def update_resource_tags(self, resource_id: str, cleanup_days: int, tags: list):
+        """
+        This method updates the resource tags
+        :param resource_id:
+        :type resource_id:
+        :param cleanup_days:
+        :type cleanup_days:
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int,
+                                   days_to_delete_resource: int = None, **kwargs):
+        """
+        This method verify and delete the resource by calculating the days
+        :return:
+        :rtype:
+        """
+        if not days_to_delete_resource:
+            days_to_delete_resource = self._days_to_take_action
+        cleanup_resources = False
+        if clean_up_days >= self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL:
+            if clean_up_days == self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL:
+                kwargs['delta_cost'] = kwargs.get('extra_purse')
+                # @Todo, If require add email alert
+            else:
+                if clean_up_days >= days_to_delete_resource:
+                    if self._dry_run == 'no':
+                        if self.get_skip_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP'):
+                            self._delete_resource(resource_id=resource_id)
+            cleanup_resources = True
+        return cleanup_resources

cloud_governance/common/helpers/json_datetime_encoder.py

@@ -0,0 +1,10 @@
+import json
+import datetime
+
+
+class JsonDateTimeEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
+            # Serialize datetime objects to ISO 8601 format
+            return obj.isoformat()
+        return super(JsonDateTimeEncoder, self).default(obj)

cloud_governance/main/aws_main_operations.py

@@ -0,0 +1,41 @@
+import os
+
+from cloud_governance.main.environment_variables import environment_variables
+from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner
+
+
+class AWSMainOperations:
+
+    def __init__(self):
+        self.__environment_variables_dict = environment_variables.environment_variables_dict
+        self.__policy = self.__environment_variables_dict.get('policy', '')
+        self.__policy_runner = PolicyRunner()
+
+    def __get_policies(self) -> dict:
+        """
+        This method gets the aws policies
+        :return:
+        :rtype:
+        """
+        policies = {}
+        policies_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'policy', 'aws')
+        for (dirpath, dirnames, filenames) in os.walk(policies_path):
+            immediate_parent = dirpath.split("/")[-1]
+            for filename in filenames:
+                if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')):
+                    policies.setdefault(immediate_parent, []).append(os.path.splitext(filename)[0])
+        return policies
+
+    def run(self):
+        """
+        This method run the AWS Policy operations
+        :return:
+        :rtype:
+        """
+        policies_list = self.__get_policies()
+        for policy_type, policies in policies_list.items():
+            # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
+            if self.__policy in policies and self.__policy == "ec2_run":
+                self.__policy_runner.run(source=policy_type)
+                return True
+        return False

cloud_governance/main/environment_variables.py

@@ -21,7 +21,8 @@ def __init__(self):
         # .env.generated can be auto-generated (by an external tool) based on the local cluster's configuration.
         for env in ".env", ".env.generated":
             try:
-                with open(env) as f:
+                file_path = os.path.join(os.path.dirname(__file__), env)
+                with open(file_path) as f:
                     for line in f.readlines():
                         key, found, value = line.strip().partition("=")
                         if not found:
@@ -40,6 +41,9 @@ def __init__(self):
         self._environment_variables_dict['account'] = EnvironmentVariables.get_env('account', '').upper().strip()
         self._environment_variables_dict['AWS_DEFAULT_REGION'] = EnvironmentVariables.get_env('AWS_DEFAULT_REGION', '')
         self._environment_variables_dict['log_level'] = EnvironmentVariables.get_env('log_level', 'INFO')
+
+        self._environment_variables_dict['DAYS_TO_TAKE_ACTION'] = int(EnvironmentVariables.get_env('DAYS_TO_TAKE_ACTION', "7"))
+
         self._environment_variables_dict['PRINT_LOGS'] = EnvironmentVariables.get_boolean_from_environment('PRINT_LOGS', True)
         if not self._environment_variables_dict['AWS_DEFAULT_REGION']:
             self._environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2'
@@ -48,7 +52,7 @@ def __init__(self):
             self._environment_variables_dict['account'] = self.get_aws_account_alias_name().upper().replace('OPENSHIFT-', '')
         self._environment_variables_dict['policy'] = EnvironmentVariables.get_env('policy', '')
 
-        self._environment_variables_dict['aws_non_cluster_policies'] = ['ec2_idle', 'ec2_stop', 'ec2_run', 'ebs_in_use',
+        self._environment_variables_dict['aws_non_cluster_policies'] = ['ec2_idle', 'ec2_stop', 'ebs_in_use',
                                                                         'ebs_unattached', 's3_inactive',
                                                                         'empty_roles', 'ip_unattached',
                                                                         'unused_nat_gateway',
@@ -84,7 +88,7 @@ def __init__(self):
         self._environment_variables_dict['end_date'] = EnvironmentVariables.get_env('end_date', '')
         self._environment_variables_dict['granularity'] = EnvironmentVariables.get_env('granularity', 'DAILY')
         self._environment_variables_dict['cost_explorer_tags'] = EnvironmentVariables.get_env('cost_explorer_tags', '{}')
-        self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = EnvironmentVariables.get_env('PUBLIC_CLOUD_NAME', 'AWS')
+        self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = EnvironmentVariables.get_env('PUBLIC_CLOUD_NAME', '')
 
         # AZURE Credentials
         self._environment_variables_dict['AZURE_ACCOUNT_ID'] = EnvironmentVariables.get_env('AZURE_ACCOUNT_ID', '')