From a269bbd729f5448ba3496db96e6e2c1b41a40b93 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Sat, 16 Dec 2023 15:36:55 +0530 Subject: [PATCH] Added the ec2_run policy --- README.md | 2 +- .../common/clouds/aws/ec2/ec2_operations.py | 8 + .../common/clouds/aws/s3/s3_operations.py | 13 +- cloud_governance/common/helpers/__init__.py | 0 .../common/helpers/aws/__init__.py | 0 .../helpers/aws/aws_cleanup_operations.py | 145 +++++++ .../common/helpers/cleanup_operations.py | 111 ++++++ .../common/helpers/json_datetime_encoder.py | 10 + cloud_governance/main/aws_main_operations.py | 41 ++ .../main/environment_variables.py | 10 +- cloud_governance/main/main.py | 234 +++++------ .../policy/aws/cleanup/__init__.py | 0 .../policy/aws/cleanup/ec2_run.py | 86 +++++ cloud_governance/policy/aws/ec2_run.py | 37 -- .../run_zombie_non_cluster_policies.py | 8 +- .../zombie_non_cluster_polices.py | 53 ++- .../policy/policy_runners/aws/__init__.py | 0 .../policy_runners/aws/policy_runner.py | 46 +++ .../policy/policy_runners/aws/upload_s3.py | 28 ++ .../policy/policy_runners/common/__init__.py | 0 .../common/abstract_policy_runner.py | 22 ++ .../policy_runners/common/abstract_upload.py | 24 ++ .../policy_runners/elasticsearch/__init__.py | 0 .../elasticsearch/upload_elastic_search.py | 40 ++ docs/source/index.md | 2 +- .../clouds/aws/daily/policies/run_policies.py | 1 + .../test_tag_non_cluster_resources.py | 2 +- .../aws/zombie_non_cluster/test_ebs_in_use.py | 5 +- .../aws/zombie_non_cluster/test_ec2_idle.py | 3 + .../aws/zombie_non_cluster/test_ec2_run.py | 47 --- .../clouds/aws/s3/test_s3_operations.py | 67 +++- .../common/helpers/__init__.py | 0 .../common/helpers/aws/__init__.py | 0 .../helpers/aws/test_aws_cleaup_operations.py | 215 +++++++++++ .../common/helpers/test_cleanup_operations.py | 92 +++++ .../helpers/test_json_datetime_encoder.py | 11 + .../policy/aws/cleanup/__init__.py | 0 .../policy/aws/cleanup/test_ec2_run.py | 365 ++++++++++++++++++ .../policy/policy_runners/__init__.py | 0 .../policy/policy_runners/aws/__init__.py | 0 .../policy_runners/aws/test_upload_s3.py | 53 +++ 41 files changed, 1539 insertions(+), 242 deletions(-) create mode 100644 cloud_governance/common/helpers/__init__.py create mode 100644 cloud_governance/common/helpers/aws/__init__.py create mode 100644 cloud_governance/common/helpers/aws/aws_cleanup_operations.py create mode 100644 cloud_governance/common/helpers/cleanup_operations.py create mode 100644 cloud_governance/common/helpers/json_datetime_encoder.py create mode 100644 cloud_governance/main/aws_main_operations.py create mode 100644 cloud_governance/policy/aws/cleanup/__init__.py create mode 100644 cloud_governance/policy/aws/cleanup/ec2_run.py delete mode 100644 cloud_governance/policy/aws/ec2_run.py create mode 100644 cloud_governance/policy/policy_runners/aws/__init__.py create mode 100644 cloud_governance/policy/policy_runners/aws/policy_runner.py create mode 100644 cloud_governance/policy/policy_runners/aws/upload_s3.py create mode 100644 cloud_governance/policy/policy_runners/common/__init__.py create mode 100644 cloud_governance/policy/policy_runners/common/abstract_policy_runner.py create mode 100644 cloud_governance/policy/policy_runners/common/abstract_upload.py create mode 100644 cloud_governance/policy/policy_runners/elasticsearch/__init__.py create mode 100644 cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py delete mode 100644 tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ec2_run.py create mode 100644 tests/unittest/cloud_governance/common/helpers/__init__.py create mode 100644 tests/unittest/cloud_governance/common/helpers/aws/__init__.py create mode 100644 tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py create mode 100644 tests/unittest/cloud_governance/common/helpers/test_cleanup_operations.py create mode 100644 tests/unittest/cloud_governance/common/helpers/test_json_datetime_encoder.py create mode 100644 tests/unittest/cloud_governance/policy/aws/cleanup/__init__.py create mode 100644 tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py create mode 100644 tests/unittest/cloud_governance/policy/policy_runners/__init__.py create mode 100644 tests/unittest/cloud_governance/policy/policy_runners/aws/__init__.py create mode 100644 tests/unittest/cloud_governance/policy/policy_runners/aws/test_upload_s3.py diff --git a/README.md b/README.md index b07fdd563..29943d412 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ This tool support the following policies: * Real time Openshift Cluster cost, User cost * [ec2_idle](cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb. -* [ec2_run](cloud_governance/policy/aws/ec2_run.py): running ec2. +* [ec2_run](cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2. * [ebs_unattached](cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status. * [ebs_in_use](cloud_governance/policy/aws/ebs_in_use.py): in use volumes. * [tag_resources](cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags diff --git a/cloud_governance/common/clouds/aws/ec2/ec2_operations.py b/cloud_governance/common/clouds/aws/ec2/ec2_operations.py index ab704ba09..1934cf1bd 100644 --- a/cloud_governance/common/clouds/aws/ec2/ec2_operations.py +++ b/cloud_governance/common/clouds/aws/ec2/ec2_operations.py @@ -656,3 +656,11 @@ def describe_tags(self, **kwargs): ec2_service_tags = self.ec2_client.describe_tags(NextToken=ec2_service_tags.get('NextToken'), **kwargs) tags_list.extend(ec2_service_tags.get('Tags', [])) return tags_list + + def get_running_instance(self): + """ + This method returns the EC2 running instances + :return: + :rtype: + """ + return self.get_ec2_instance_list(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}]) diff --git a/cloud_governance/common/clouds/aws/s3/s3_operations.py b/cloud_governance/common/clouds/aws/s3/s3_operations.py index f25731ba9..d3a5ef14e 100644 --- a/cloud_governance/common/clouds/aws/s3/s3_operations.py +++ b/cloud_governance/common/clouds/aws/s3/s3_operations.py @@ -242,7 +242,8 @@ def get_last_objects(self, bucket: str, logs_bucket_key: str = '', policy: str = date_key = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y/%m/%d") key_prefix = f'{logs_bucket_key}/{policy}/{date_key}' objs = self.__s3_client.list_objects_v2(Bucket=bucket, Prefix=key_prefix)['Contents'] - except: + except Exception as err: + print(err) return None get_last_modified_key = lambda obj: int(obj['LastModified'].strftime('%s')) full_path = [obj['Key'] for obj in sorted(objs, key=get_last_modified_key)][-1] @@ -294,7 +295,7 @@ def find_bucket(self, bucket_name: str): return True return False - def __get_s3_latest_policy_file(self, policy: str): + def __get_s3_latest_policy_file(self, policy: str, key_prefix: str = ''): """ This method return latest policy logs @param policy: @@ -302,9 +303,11 @@ def __get_s3_latest_policy_file(self, policy: str): """ return self.get_last_objects(bucket=self.__bucket, logs_bucket_key=f'{self.__logs_bucket_key}/{self.__region}', + key_prefix=key_prefix, policy=policy) - def get_last_s3_policy_content(self, policy: str = '', file_name: str = '', s3_file_path: str = None): + def get_last_s3_policy_content(self, policy: str = '', file_name: str = '', s3_file_path: str = None, + key_prefix: str = ''): """ This method return last policy content @return: @@ -312,8 +315,8 @@ def get_last_s3_policy_content(self, policy: str = '', file_name: str = '', s3_f with tempfile.TemporaryDirectory() as temp_local_directory: local_file = temp_local_directory + '/' + file_name + '.gz' if not s3_file_path: - if self.__get_s3_latest_policy_file(policy=policy): - s3_file_path = self.__get_s3_latest_policy_file(policy=policy) + if self.__get_s3_latest_policy_file(policy=policy, key_prefix=key_prefix): + s3_file_path = self.__get_s3_latest_policy_file(policy=policy, key_prefix=key_prefix) self.download_file(bucket=self.__bucket, key=str(s3_file_path), download_file=file_name + '.gz', diff --git a/cloud_governance/common/helpers/__init__.py b/cloud_governance/common/helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/common/helpers/aws/__init__.py b/cloud_governance/common/helpers/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/common/helpers/aws/aws_cleanup_operations.py b/cloud_governance/common/helpers/aws/aws_cleanup_operations.py new file mode 100644 index 000000000..4539ade50 --- /dev/null +++ b/cloud_governance/common/helpers/aws/aws_cleanup_operations.py @@ -0,0 +1,145 @@ +import datetime + +import boto3 + +from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations +from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations +from cloud_governance.common.logger.init_logger import logger + + +class AWSCleanUpOperations(AbstractCleanUpOperations): + + def __init__(self): + super().__init__() + self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') + self.__s3operations = S3Operations(region_name=self._region) + self._ec2_client = boto3.client('ec2', region_name=self._region) + self._s3_client = boto3.client('s3') + self._iam_client = boto3.client('iam') + + def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str: + """ + This method returns the tag value from the tags + :param tags: + :type tags: + :param tag_name: + :type tag_name: + :return: + :rtype: + """ + if tags: + for tag in tags: + if tag.get('Key').strip().lower() == tag_name.lower(): + return tag.get('Value').strip() + return '' + + def get_clean_up_days_count(self, tags: list): + """ + This method returns the cleanup days count + :param tags: + :type tags: + :return: + :rtype: + """ + last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount') + if not last_used_day: + return 1 + else: + date, days = last_used_day.split('@') + if date != str(self.CURRENT_DATE): + return int(days) + 1 + return 1 if int(days) == 0 else int(days) + + def _delete_resource(self, resource_id: str): + """ + This method deletes the resource by verifying the policy + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + action = "deleted" + try: + if self._policy == 's3_inactive': + self._s3_client.delete_bucket(Bucket=resource_id) + elif self._policy == 'empty_roles': + self._iam_client.delete_role(RoleName=resource_id) + elif self._policy == 'ebs_unattached': + self._ec2_client.delete_volume(VolumeId=resource_id) + elif self._policy == 'ip_unattached': + self._ec2_client.release_address(AllocationId=resource_id) + elif self._policy == 'unused_nat_gateway': + self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id) + elif self._policy == 'zombie_snapshots': + self._ec2_client.delete_snapshot(SnapshotId=resource_id) + elif self._policy == 'ec2_run': + self._ec2_client.stop_instances(InstanceIds=[resource_id]) + action = "Stopped" + logger.info(f'{self._policy} {action}: {resource_id}') + except Exception as err: + logger.info(f'Exception raised: {err}: {resource_id}') + + def __remove_tag_key_aws(self, tags: list): + """ + This method returns the tags that does not contain key startswith aws: + :param tags: + :type tags: + :return: + :rtype: + """ + custom_tags = [] + for tag in tags: + if not tag.get('Key').lower().startswith('aws'): + custom_tags.append(tag) + return custom_tags + + def __update_tag_value(self, tags: list, tag_name: str, tag_value: str): + """ + This method updates the tag_value + @param tags: + @param tag_name: + @param tag_value: + @return: + """ + if self._dry_run == "yes": + tag_value = 0 + tag_value = f'{self.CURRENT_DATE}@{tag_value}' + found = False + if tags: + for tag in tags: + if tag.get('Key') == tag_name: + if tag.get('Value').split("@")[0] != self.CURRENT_DATE: + tag['Value'] = tag_value + else: + if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1: + tag['Value'] = tag_value + found = True + if not found: + tags.append({'Key': tag_name, 'Value': tag_value}) + tags = self.__remove_tag_key_aws(tags=tags) + return tags + + def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list, force_tag_update: str = ''): + """ + This method updates the resource tags + :param force_tag_update: + :type force_tag_update: + :param tags: + :type tags: + :param cleanup_days: + :type cleanup_days: + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) + try: + if self._policy == 's3_inactive': + self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags}) + elif self._policy == 'empty_roles': + self._iam_client.tag_role(RoleName=resource_id, Tags=tags) + elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', 'ec2_run'): + self._ec2_client.create_tags(Resources=[resource_id], Tags=tags) + except Exception as err: + logger.info(f'Exception raised: {err}: {resource_id}') diff --git a/cloud_governance/common/helpers/cleanup_operations.py b/cloud_governance/common/helpers/cleanup_operations.py new file mode 100644 index 000000000..aa8ac1521 --- /dev/null +++ b/cloud_governance/common/helpers/cleanup_operations.py @@ -0,0 +1,111 @@ +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Union + +from cloud_governance.main.environment_variables import environment_variables + + +class AbstractCleanUpOperations(ABC): + + DAYS_TO_NOTIFY_ADMINS = 2 + DAYS_TO_TRIGGER_RESOURCE_MAIL = 4 + DAILY_HOURS = 24 + CURRENT_DATE = datetime.utcnow().date().__str__() + + def __init__(self): + self._environment_variables_dict = environment_variables.environment_variables_dict + self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION') + self._dry_run = self._environment_variables_dict.get('dry_run') + self._policy = self._environment_variables_dict.get('policy') + self._force_delete = self._environment_variables_dict.get('FORCE_DELETE') + self._resource_id = self._environment_variables_dict.get('RESOURCE_ID') + + @abstractmethod + def get_clean_up_days_count(self, tags: Union[list, dict]): + """ + This method returns the cleanup days count + :param tags: + :type tags: + :return: + :rtype: + """ + raise NotImplementedError("This method is Not yet implemented") + + @abstractmethod + def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str): + """ + This method returns the tag_value from the tags + :param tags: + :type tags: + :param tag_name: + :type tag_name: + :return: + :rtype: + """ + raise NotImplementedError("This method is Not yet implemented") + + def get_skip_policy_value(self, tags: Union[list, dict]) -> str: + """ + This method returns the skip value + :param tags: + :type tags: + :return: + :rtype: + """ + policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Policy').strip() + if not policy_value: + policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Skip').strip() + if policy_value: + return policy_value.replace('_', '').replace('-', '').upper() + return 'NA' + + @abstractmethod + def _delete_resource(self, resource_id: str): + """ + This method deletes the resource + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + raise NotImplementedError("This method is Not yet implemented") + + @abstractmethod + def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list): + """ + This method updates the resource tags + :param resource_id: + :type resource_id: + :param cleanup_days: + :type cleanup_days: + :param tags: + :type tags: + :return: + :rtype: + """ + raise NotImplementedError("This method is Not yet implemented") + + def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int, + days_to_delete_resource: int = None, **kwargs): + """ + This method verify and delete the resource by calculating the days + :return: + :rtype: + """ + if self._resource_id == resource_id and self._force_delete and self._dry_run == 'no': + self._delete_resource(resource_id=resource_id) + return True + if not days_to_delete_resource: + days_to_delete_resource = self._days_to_take_action + cleanup_resources = False + if clean_up_days >= self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL: + if clean_up_days == self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL: + kwargs['delta_cost'] = kwargs.get('extra_purse') + # @Todo, If it require add email alert. May In future will add the email alert. + else: + if clean_up_days >= days_to_delete_resource: + if self._dry_run == 'no': + if self.get_skip_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP'): + self._delete_resource(resource_id=resource_id) + cleanup_resources = True + return cleanup_resources diff --git a/cloud_governance/common/helpers/json_datetime_encoder.py b/cloud_governance/common/helpers/json_datetime_encoder.py new file mode 100644 index 000000000..5d4103cf0 --- /dev/null +++ b/cloud_governance/common/helpers/json_datetime_encoder.py @@ -0,0 +1,10 @@ +import json +import datetime + + +class JsonDateTimeEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)): + # Serialize datetime objects to ISO 8601 format + return obj.isoformat() + return super(JsonDateTimeEncoder, self).default(obj) diff --git a/cloud_governance/main/aws_main_operations.py b/cloud_governance/main/aws_main_operations.py new file mode 100644 index 000000000..60823ff31 --- /dev/null +++ b/cloud_governance/main/aws_main_operations.py @@ -0,0 +1,41 @@ +import os + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner + + +class AWSMainOperations: + + def __init__(self): + self.__environment_variables_dict = environment_variables.environment_variables_dict + self.__policy = self.__environment_variables_dict.get('policy', '') + self.__policy_runner = PolicyRunner() + + def __get_policies(self) -> dict: + """ + This method gets the aws policies + :return: + :rtype: + """ + policies = {} + policies_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'policy', 'aws') + for (dirpath, dirnames, filenames) in os.walk(policies_path): + immediate_parent = dirpath.split("/")[-1] + for filename in filenames: + if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')): + policies.setdefault(immediate_parent, []).append(os.path.splitext(filename)[0]) + return policies + + def run(self): + """ + This method run the AWS Policy operations + :return: + :rtype: + """ + policies_list = self.__get_policies() + for policy_type, policies in policies_list.items(): + # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement + if self.__policy in policies and self.__policy == "ec2_run": + self.__policy_runner.run(source=policy_type) + return True + return False diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index 9666f3dc7..8faca2524 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -21,7 +21,8 @@ def __init__(self): # .env.generated can be auto-generated (by an external tool) based on the local cluster's configuration. for env in ".env", ".env.generated": try: - with open(env) as f: + file_path = os.path.join(os.path.dirname(__file__), env) + with open(file_path) as f: for line in f.readlines(): key, found, value = line.strip().partition("=") if not found: @@ -40,6 +41,9 @@ def __init__(self): self._environment_variables_dict['account'] = EnvironmentVariables.get_env('account', '').upper().strip() self._environment_variables_dict['AWS_DEFAULT_REGION'] = EnvironmentVariables.get_env('AWS_DEFAULT_REGION', '') self._environment_variables_dict['log_level'] = EnvironmentVariables.get_env('log_level', 'INFO') + + self._environment_variables_dict['DAYS_TO_TAKE_ACTION'] = int(EnvironmentVariables.get_env('DAYS_TO_TAKE_ACTION', "7")) + self._environment_variables_dict['PRINT_LOGS'] = EnvironmentVariables.get_boolean_from_environment('PRINT_LOGS', True) if not self._environment_variables_dict['AWS_DEFAULT_REGION']: self._environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2' @@ -48,7 +52,7 @@ def __init__(self): self._environment_variables_dict['account'] = self.get_aws_account_alias_name().upper().replace('OPENSHIFT-', '') self._environment_variables_dict['policy'] = EnvironmentVariables.get_env('policy', '') - self._environment_variables_dict['aws_non_cluster_policies'] = ['ec2_idle', 'ec2_stop', 'ec2_run', 'ebs_in_use', + self._environment_variables_dict['aws_non_cluster_policies'] = ['ec2_idle', 'ec2_stop', 'ebs_in_use', 'ebs_unattached', 's3_inactive', 'empty_roles', 'ip_unattached', 'unused_nat_gateway', @@ -84,7 +88,7 @@ def __init__(self): self._environment_variables_dict['end_date'] = EnvironmentVariables.get_env('end_date', '') self._environment_variables_dict['granularity'] = EnvironmentVariables.get_env('granularity', 'DAILY') self._environment_variables_dict['cost_explorer_tags'] = EnvironmentVariables.get_env('cost_explorer_tags', '{}') - self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = EnvironmentVariables.get_env('PUBLIC_CLOUD_NAME', 'AWS') + self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = EnvironmentVariables.get_env('PUBLIC_CLOUD_NAME', '') # AZURE Credentials self._environment_variables_dict['AZURE_ACCOUNT_ID'] = EnvironmentVariables.get_env('AZURE_ACCOUNT_ID', '') diff --git a/cloud_governance/main/main.py b/cloud_governance/main/main.py index ca5bd9624..2f7c857ee 100644 --- a/cloud_governance/main/main.py +++ b/cloud_governance/main/main.py @@ -4,6 +4,7 @@ import boto3 # regions from cloud_governance.cloud_resource_orchestration.monitor.cloud_monitor import CloudMonitor +from cloud_governance.main.aws_main_operations import AWSMainOperations from cloud_governance.main.main_common_operations import run_common_policies from cloud_governance.main.run_cloud_resource_orchestration import run_cloud_resource_orchestration from cloud_governance.policy.policy_operations.aws.cost_expenditure.cost_report_policies import CostReportPolicies @@ -170,6 +171,19 @@ def run_policy(account: str, policy: str, region: str, dry_run: str): raise Exception(f'Missing Policy name: {policy}') +def is_policy_aws(): + """ + This method true if the env contain AWS variables like , (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY) + or PUBLIC_CLOUD_NAME=AWS + :return: + :rtype: + """ + aws_access_key = environment_variables_dict.get('AWS_ACCESS_KEY_ID', '') + aws_secret_key = environment_variables_dict.get('AWS_SECRET_ACCESS_KEY', '') + public_cloud_name = environment_variables_dict.get('PUBLIC_CLOUD_NAME', '') + return (aws_access_key and aws_secret_key) or (public_cloud_name.lower() == 'aws') + + @logger_time_stamp def main(): """ @@ -190,125 +204,129 @@ def main(): es_index = environment_variables_dict.get('es_index', '') es_doc_type = environment_variables_dict.get('es_doc_type', '') bucket = environment_variables_dict.get('bucket', '') + response = False + if is_policy_aws(): + aws_main_operations = AWSMainOperations() + response = aws_main_operations.run() + if not response: + if environment_variables_dict.get('COMMON_POLICIES'): + run_common_policies() + elif environment_variables_dict.get('CLOUD_RESOURCE_ORCHESTRATION'): + run_cloud_resource_orchestration() + else: + non_cluster_polices_runner = None + is_non_cluster_polices_runner = policy in environment_variables_dict.get('aws_non_cluster_policies') + if is_non_cluster_polices_runner: + non_cluster_polices_runner = ZombieNonClusterPolicies() - if environment_variables_dict.get('COMMON_POLICIES'): - run_common_policies() - elif environment_variables_dict.get('CLOUD_RESOURCE_ORCHESTRATION'): - run_cloud_resource_orchestration() - else: - non_cluster_polices_runner = None - is_non_cluster_polices_runner = policy in environment_variables_dict.get('aws_non_cluster_policies') - if is_non_cluster_polices_runner: - non_cluster_polices_runner = ZombieNonClusterPolicies() - - ibm_classic_infrastructure_policy_runner = None - is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('ibm_policies') - if not is_tag_ibm_classic_infrastructure_runner: - if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'IBM': - is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('cost_policies') - if is_tag_ibm_classic_infrastructure_runner: - ibm_classic_infrastructure_policy_runner = IBMPolicyRunner() + ibm_classic_infrastructure_policy_runner = None + is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('ibm_policies') + if not is_tag_ibm_classic_infrastructure_runner: + if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'IBM': + is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('cost_policies') + if is_tag_ibm_classic_infrastructure_runner: + ibm_classic_infrastructure_policy_runner = IBMPolicyRunner() - is_cost_explorer_policies_runner = '' - if environment_variables_dict.get('PUBLIC_CLOUD_NAME') == 'AWS': - cost_explorer_policies_runner = None - is_cost_explorer_policies_runner = policy in environment_variables_dict.get('cost_policies') - if is_cost_explorer_policies_runner: - cost_explorer_policies_runner = CostReportPolicies() + is_cost_explorer_policies_runner = '' + if environment_variables_dict.get('PUBLIC_CLOUD_NAME') == 'AWS': + cost_explorer_policies_runner = None + is_cost_explorer_policies_runner = policy in environment_variables_dict.get('cost_policies') + if is_cost_explorer_policies_runner: + cost_explorer_policies_runner = CostReportPolicies() - is_azure_policy_runner = '' - if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AZURE': - azure_cost_policy_runner = None - is_azure_policy_runner = policy in environment_variables_dict.get('cost_policies') - if is_azure_policy_runner: - azure_cost_policy_runner = AzurePolicyRunner() + is_azure_policy_runner = '' + if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AZURE': + azure_cost_policy_runner = None + is_azure_policy_runner = policy in environment_variables_dict.get('cost_policies') + if is_azure_policy_runner: + azure_cost_policy_runner = AzurePolicyRunner() - is_gcp_policy_runner = '' - if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'GCP': - gcp_cost_policy_runner = None - is_gcp_policy_runner = policy in environment_variables_dict.get('cost_policies') - if is_gcp_policy_runner: - gcp_cost_policy_runner = GcpPolicyRunner() + is_gcp_policy_runner = '' + if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'GCP': + gcp_cost_policy_runner = None + is_gcp_policy_runner = policy in environment_variables_dict.get('cost_policies') + if is_gcp_policy_runner: + gcp_cost_policy_runner = GcpPolicyRunner() - @logger_time_stamp - def run_non_cluster_polices_runner(): - """ - This method run the aws non-cluster policies - @return: - """ - non_cluster_polices_runner.run() + @logger_time_stamp + def run_non_cluster_polices_runner(): + """ + This method run the aws non-cluster policies + @return: + """ + non_cluster_polices_runner.run() - def run_tag_ibm_classic_infrastructure_runner(): - """ - This method run the IBM policies - @return: - """ - ibm_classic_infrastructure_policy_runner.run() + def run_tag_ibm_classic_infrastructure_runner(): + """ + This method run the IBM policies + @return: + """ + ibm_classic_infrastructure_policy_runner.run() - @logger_time_stamp - def run_cost_explorer_policies_runner(): - """ - This method run the aws cost_explorer policies - @return: - """ - cost_explorer_policies_runner.run() + @logger_time_stamp + def run_cost_explorer_policies_runner(): + """ + This method run the aws cost_explorer policies + @return: + """ + cost_explorer_policies_runner.run() - @logger_time_stamp - def run_azure_policy_runner(): - """ - This method run the azure policies - @return: - """ - azure_cost_policy_runner.run() + @logger_time_stamp + def run_azure_policy_runner(): + """ + This method run the azure policies + @return: + """ + azure_cost_policy_runner.run() - @logger_time_stamp - def run_gcp_policy_runner(): - """ - This method run the gcp policies - """ - gcp_cost_policy_runner.run() + @logger_time_stamp + def run_gcp_policy_runner(): + """ + This method run the gcp policies + """ + gcp_cost_policy_runner.run() - # 1. ELK Uploader - if upload_data_es: - input_data = {'es_host': es_host, - 'es_port': int(es_port), - 'es_index': es_index, - 'es_doc_type': es_doc_type, - 'es_add_items': {'account': account}, - 'bucket': bucket, - 'logs_bucket_key': 'logs', - 's3_file_name': 'resources.json', - 'region': region_env, - 'policy': policy, - } - elk_uploader = ESUploader(**input_data) - elk_uploader.upload_to_es(account=account) - # 2. POLICY - elif is_non_cluster_polices_runner: - run_non_cluster_polices_runner() - elif is_tag_ibm_classic_infrastructure_runner: - run_tag_ibm_classic_infrastructure_runner() - elif is_cost_explorer_policies_runner: - run_cost_explorer_policies_runner() - elif is_azure_policy_runner: - run_azure_policy_runner() - elif is_gcp_policy_runner: - run_gcp_policy_runner() - else: - if not policy: - logger.exception(f'Missing Policy name: "{policy}"') - raise Exception(f'Missing Policy name: "{policy}"') - if region_env == 'all': - # must be set for boto3 client default region - # environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2' - ec2 = boto3.client('ec2') - regions_data = ec2.describe_regions() - for region in regions_data['Regions']: - # logger.info(f"region: {region['RegionName']}") - environment_variables_dict['AWS_DEFAULT_REGION'] = region['RegionName'] - run_policy(account=account, policy=policy, region=region['RegionName'], dry_run=dry_run) + # 1. ELK Uploader + if upload_data_es: + input_data = {'es_host': es_host, + 'es_port': int(es_port), + 'es_index': es_index, + 'es_doc_type': es_doc_type, + 'es_add_items': {'account': account}, + 'bucket': bucket, + 'logs_bucket_key': 'logs', + 's3_file_name': 'resources.json', + 'region': region_env, + 'policy': policy, + } + elk_uploader = ESUploader(**input_data) + elk_uploader.upload_to_es(account=account) + # 2. POLICY + elif is_non_cluster_polices_runner: + run_non_cluster_polices_runner() + elif is_tag_ibm_classic_infrastructure_runner: + run_tag_ibm_classic_infrastructure_runner() + elif is_cost_explorer_policies_runner: + run_cost_explorer_policies_runner() + elif is_azure_policy_runner: + run_azure_policy_runner() + elif is_gcp_policy_runner: + run_gcp_policy_runner() else: - run_policy(account=account, policy=policy, region=region_env, dry_run=dry_run) + if not policy: + logger.exception(f'Missing Policy name: "{policy}"') + raise Exception(f'Missing Policy name: "{policy}"') + if region_env == 'all': + # must be set for boto3 client default region + # environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2' + ec2 = boto3.client('ec2') + regions_data = ec2.describe_regions() + for region in regions_data['Regions']: + # logger.info(f"region: {region['RegionName']}") + environment_variables_dict['AWS_DEFAULT_REGION'] = region['RegionName'] + run_policy(account=account, policy=policy, region=region['RegionName'], dry_run=dry_run) + else: + run_policy(account=account, policy=policy, region=region_env, dry_run=dry_run) main() diff --git a/cloud_governance/policy/aws/cleanup/__init__.py b/cloud_governance/policy/aws/cleanup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/policy/aws/cleanup/ec2_run.py b/cloud_governance/policy/aws/cleanup/ec2_run.py new file mode 100644 index 000000000..4b6ee19f3 --- /dev/null +++ b/cloud_governance/policy/aws/cleanup/ec2_run.py @@ -0,0 +1,86 @@ +import datetime + +from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import \ + NonClusterZombiePolicy + + +class EC2Run(NonClusterZombiePolicy): + + RESOURCE_ACTION = "Stopped" + + def __init__(self): + super(EC2Run, self).__init__() + self.__es_index = 'cloud-governance-ec2-instance-types' + + def __update_instance_type_count(self, instances: list): + """ + This method updates the instance types count + :param instances: + :type instances: + :return: + :rtype: + """ + instance_types = {} + for instance in instances: + instance_type = instance.get('InstanceType') + instance_types[instance_type] = instance_types.get(instance_type, 0) + 1 + es_instance_types_data = [] + for key, value in instance_types.items(): + es_instance_types_data.append({ + 'instance_type': key, + 'instance_count': value, + 'timestamp': datetime.datetime.utcnow(), + 'region': self._region, + 'account': self._account.upper().replace('OPENSHIFT-', ''), + 'index_id': f'{key}-{self._account.lower()}-{self._region}-{str(datetime.datetime.utcnow().date())}' + }) + self._es_upload.es_upload_data(items=es_instance_types_data, es_index=self.__es_index, set_index='index_id') + + def __ec2_run(self): + """ + This method list the running instances and upload to elastic_search + :return: + :rtype: + """ + instances = self._ec2_operations.get_ec2_instance_list() + self.__update_instance_type_count(instances=instances) + running_instances_data = [] + for instance in instances: + tags = instance.get('Tags', []) + if instance.get('State', {}).get('Name') == 'running': + running_days = self._calculate_days(instance.get('LaunchTime')) + cleanup_days = self._aws_cleanup_policies.get_clean_up_days_count(tags=tags) + cleanup_result = self._aws_cleanup_policies.verify_and_delete_resource( + resource_id=instance.get('InstanceId'), tags=tags, + clean_up_days=cleanup_days) + resource_data = { + 'ResourceId': instance.get('InstanceId'), + 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'), + 'SkipPolicy': self._aws_cleanup_policies.get_skip_policy_value(tags=tags), + 'LaunchTime': instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'InstanceType': instance.get('InstanceType'), + 'InstanceState': instance.get('State', {}).get('Name') if not cleanup_result else 'stopped', + 'StateTransitionReason': instance.get('StateTransitionReason'), + 'RunningDays': running_days, + 'CleanUpDays': cleanup_days, + 'DryRun': self._dry_run, + 'Name': self._get_tag_name_from_tags(tags=tags, tag_name='Name'), + 'RegionName': self._region, + f'Resource{self.RESOURCE_ACTION}': str(cleanup_result) + } + if self._force_delete and self._dry_run == 'no': + resource_data.update({'ForceDeleted': str(self._force_delete)}) + running_instances_data.append(resource_data) + else: + cleanup_days = 0 + self._aws_cleanup_policies.update_resource_day_count_tag(resource_id=instance.get('InstanceId'), + cleanup_days=cleanup_days, tags=tags) + + return running_instances_data + + def run(self): + """ + This method list all the running instances + @return: + """ + return self.__ec2_run() diff --git a/cloud_governance/policy/aws/ec2_run.py b/cloud_governance/policy/aws/ec2_run.py deleted file mode 100644 index 8a952f556..000000000 --- a/cloud_governance/policy/aws/ec2_run.py +++ /dev/null @@ -1,37 +0,0 @@ -import datetime - -from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy - - -class EC2Run(NonClusterZombiePolicy): - - def __init__(self): - super(EC2Run, self).__init__() - self.__es_index = 'cloud-governance-ec2-instance-types' - - def run(self): - """ - This method list all in-use ebs volumes - @return: - """ - running_instances = [] - instances = self._ec2_operations.get_instances() - instance_types = {} - for instance in instances: - for resource in instance['Instances']: - if resource.get('State').get('Name') == 'running': - running_instances.append(resource) - instance_type = resource.get('InstanceType') - instance_types[instance_type] = instance_types.get(instance_type, 0)+1 - es_instance_types_data = [] - for key, value in instance_types.items(): - es_instance_types_data.append({ - 'instance_type': key, - 'instance_count': value, - 'timestamp': datetime.datetime.utcnow(), - 'region': self._region, - 'account': self._account.upper().replace('OPENSHIFT-', ''), - 'index_id': f'{key}-{self._account.lower()}-{self._region}-{str(datetime.datetime.utcnow().date())}' - }) - self._es_upload.es_upload_data(items=es_instance_types_data, es_index=self.__es_index, set_index='index_id') - return self._organise_instance_data(running_instances) diff --git a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py index e56cf1b65..c043a9fe2 100644 --- a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py +++ b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py @@ -10,6 +10,7 @@ from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations +from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations from cloud_governance.common.ldap.ldap_search import LdapSearch from cloud_governance.common.logger.init_logger import logger from cloud_governance.common.mails.mail_message import MailMessage @@ -26,12 +27,14 @@ class NonClusterZombiePolicy: DAILY_HOURS = 24 def __init__(self): + self._aws_cleanup_policies = AWSCleanUpOperations() self.__environment_variables_dict = environment_variables.environment_variables_dict self._end_date = datetime.datetime.now() self._start_date = self._end_date - datetime.timedelta(days=self.DAYS_TO_DELETE_RESOURCE) self._account = self.__environment_variables_dict.get('account', '') self._dry_run = self.__environment_variables_dict.get('dry_run', 'yes') self._region = self.__environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') + self._force_delete = self.__environment_variables_dict.get('FORCE_DELETE') self._policy = self.__environment_variables_dict.get('policy', '') self._policy_output = self.__environment_variables_dict.get('policy_output', '') self._ec2_client = boto3.client('ec2', region_name=self._region) @@ -95,7 +98,6 @@ def _get_tag_name_from_tags(self, tags: list, tag_name: str = 'Name'): for tag in tags: if tag.get('Key').strip().lower() == tag_name.lower(): return tag.get('Value').strip() - return '' return '' def _calculate_days(self, create_date: datetime): @@ -103,7 +105,7 @@ def _calculate_days(self, create_date: datetime): This method returns the days @return: """ - today = datetime.date.today() + today = datetime.datetime.utcnow().date() days = today - create_date.date() return days.days @@ -326,7 +328,7 @@ def _organise_instance_data(self, resources: list): 'ResourceId': instance.get('InstanceId'), 'InstanceId': instance.get('InstanceId'), 'User': self._ec2_operations.get_tag_value_from_tags(tags=instance.get('Tags', []), tag_name='User'), 'Policy': skip_policy, - 'LaunchTime': instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'LaunchTime': instance.get('LaunchTime').strftime("%Y-%m-%dT%H:%M:%S+00:00"), 'InstanceType': instance.get('InstanceType'), 'InstanceState': instance.get('State', {}).get('Name'), 'StateTransitionReason': instance.get('StateTransitionReason') diff --git a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py index f03f16a5c..06226bf42 100644 --- a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py +++ b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py @@ -22,32 +22,31 @@ def run(self): for cls in inspect.getmembers(zombie_non_cluster_policy_module, inspect.isclass): if self._policy.replace('_', '') == cls[0].lower(): response = cls[1]().run() - if response: - if isinstance(response, str): - logger.info(f'key: {cls[0]}, Response: {response}') - else: - logger.info(f'key: {cls[0]}, count: {len(response)}, {response}') - policy_result = response - - if self._es_operations.check_elastic_search_connection(): - if policy_result: - if len(policy_result) > 500: - self._es_operations.upload_data_in_bulk(data_items=policy_result.copy(), - index=self._es_index) - else: - for policy_dict in policy_result: - policy_dict['region_name'] = self._region - policy_dict['account'] = self._account - self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index) - logger.info(f'Uploaded the policy results to elasticsearch index: {self._es_index}') + if isinstance(response, str): + logger.info(f'key: {cls[0]}, Response: {response}') + else: + logger.info(f'key: {cls[0]}, count: {len(response)}, {response}') + policy_result = response + + if self._es_operations.check_elastic_search_connection(): + if policy_result: + if len(policy_result) > 500: + self._es_operations.upload_data_in_bulk(data_items=policy_result.copy(), + index=self._es_index) else: - logger.error(f'No data to upload on @{self._account} at {datetime.utcnow()}') + for policy_dict in policy_result: + policy_dict['region_name'] = self._region + policy_dict['account'] = self._account + self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index) + logger.info(f'Uploaded the policy results to elasticsearch index: {self._es_index}') else: - logger.error('ElasticSearch host is not pingable, Please check ') - - if self._policy_output: - # if self._policy not in ('ec2_idle', 'ebs_in_use', 'ec2_run', 's3_inactive', 'zombie_snapshots', 'nat_gateway_unused'): - # beautify_data = self._beautify_upload_data(upload_resource_data=response) - # policy_result = {'count': len(beautify_data), self._policy: beautify_data} - logger.info(policy_result) - self._s3operations.save_results_to_s3(policy=self._policy.replace('_', '-'), policy_output=self._policy_output, policy_result=policy_result) + logger.error(f'No data to upload on @{self._account} at {datetime.utcnow()}') + else: + logger.error('ElasticSearch host is not pingable, Please check ') + + if self._policy_output: + # if self._policy not in ('ec2_idle', 'ebs_in_use', 'ec2_run', 's3_inactive', 'zombie_snapshots', 'nat_gateway_unused'): + # beautify_data = self._beautify_upload_data(upload_resource_data=response) + # policy_result = {'count': len(beautify_data), self._policy: beautify_data} + logger.info(policy_result) + self._s3operations.save_results_to_s3(policy=self._policy.replace('_', '-'), policy_output=self._policy_output, policy_result=policy_result) diff --git a/cloud_governance/policy/policy_runners/aws/__init__.py b/cloud_governance/policy/policy_runners/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/policy/policy_runners/aws/policy_runner.py b/cloud_governance/policy/policy_runners/aws/policy_runner.py new file mode 100644 index 000000000..0816ccc74 --- /dev/null +++ b/cloud_governance/policy/policy_runners/aws/policy_runner.py @@ -0,0 +1,46 @@ + +import importlib +import inspect + +from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.policy.policy_runners.common.abstract_policy_runner import AbstractPolicyRunner + + +class PolicyRunner(AbstractPolicyRunner): + + def __init__(self): + super().__init__() + self.__ec2_operations = EC2Operations() + + def run(self, source: str = "", upload: bool = True): + """ + This method run the AWS policies classes + :param upload: + :type upload: + :param source: + :type source: + :return: + :rtype: + """ + source_policy = f"{source}.{self._policy}" if source else self._policy + logger.info(f'account={self._account}, policy={self._policy}, dry_run={self._dry_run}') + zombie_non_cluster_policy_module = importlib.import_module(f'cloud_governance.policy.aws.{source_policy}') + + for cls in inspect.getmembers(zombie_non_cluster_policy_module, inspect.isclass): + if self._policy.replace('_', '').replace('-', '') == cls[0].lower(): + active_regions = [self._region] + if self._run_active_regions: + active_regions = self.__ec2_operations.get_active_regions() + logger.info("Running the policy in All AWS active regions") + for active_region in active_regions: + logger.info(f"Running the {self._policy} in Region: {active_region}") + self._environment_variables_dict['AWS_DEFAULT_REGION'] = active_region + response = cls[1]().run() + if isinstance(response, str): + logger.info(f'key: {cls[0]}, Response: {response}') + else: + logger.info(f'key: {cls[0]}, count: {len(response)}, {response}') + if upload: + self._upload_elastic_search.upload(data=response) + self._upload_to_s3.upload(data=response) diff --git a/cloud_governance/policy/policy_runners/aws/upload_s3.py b/cloud_governance/policy/policy_runners/aws/upload_s3.py new file mode 100644 index 000000000..6abd3ddc1 --- /dev/null +++ b/cloud_governance/policy/policy_runners/aws/upload_s3.py @@ -0,0 +1,28 @@ +import json +from typing import Union + +from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations +from cloud_governance.common.helpers.json_datetime_encoder import JsonDateTimeEncoder +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.policy.policy_runners.common.abstract_upload import AbstractUpload + + +class UploadS3(AbstractUpload): + + def __init__(self): + super().__init__() + self._s3operations = S3Operations(region_name=self._region) + + def upload(self, data: Union[list, dict]): + """ + This method upload data to S3 + :param data: + :type data: + :return: + :rtype: + """ + if self._policy_output: + data = json.dumps(data, cls=JsonDateTimeEncoder) + self._s3operations.save_results_to_s3(policy=self._policy.replace('_', '-'), + policy_output=self._policy_output, policy_result=data) + logger.info(f"Uploaded the data s3 Bucket: {self._policy_output}") diff --git a/cloud_governance/policy/policy_runners/common/__init__.py b/cloud_governance/policy/policy_runners/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py b/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py new file mode 100644 index 000000000..852744a4c --- /dev/null +++ b/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py @@ -0,0 +1,22 @@ +from abc import abstractmethod, ABC + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.policy_runners.aws.upload_s3 import UploadS3 +from cloud_governance.policy.policy_runners.elasticsearch.upload_elastic_search import UploadElasticSearch + + +class AbstractPolicyRunner(ABC): + + def __init__(self): + self._environment_variables_dict = environment_variables.environment_variables_dict + self._policy = self._environment_variables_dict.get('policy', '') + self._account = self._environment_variables_dict.get('account', '') + self._dry_run = self._environment_variables_dict.get('dry_run', 'yes') + self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') + self._run_active_regions = self._environment_variables_dict.get('RUN_ACTIVE_REGIONS') + self._upload_to_s3 = UploadS3() + self._upload_elastic_search = UploadElasticSearch() + + @abstractmethod + def run(self): + raise NotImplementedError("This method is not yet implemented") diff --git a/cloud_governance/policy/policy_runners/common/abstract_upload.py b/cloud_governance/policy/policy_runners/common/abstract_upload.py new file mode 100644 index 000000000..5baf878a3 --- /dev/null +++ b/cloud_governance/policy/policy_runners/common/abstract_upload.py @@ -0,0 +1,24 @@ +from abc import ABC, abstractmethod +from typing import Union + +from cloud_governance.main.environment_variables import environment_variables + + +class AbstractUpload(ABC): + + def __init__(self): + self._environment_variables_dict = environment_variables.environment_variables_dict + self._account = self._environment_variables_dict.get('account', '') + self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') + self._es_index = self._environment_variables_dict.get('es_index') + self._policy_output = self._environment_variables_dict.get('policy_output', '') + self._policy = self._environment_variables_dict.get('policy', '') + + @abstractmethod + def upload(self, data: Union[list, dict]): + """ + This method upload data + :return: + :rtype: + """ + raise NotImplemented("This is not yet implemented") diff --git a/cloud_governance/policy/policy_runners/elasticsearch/__init__.py b/cloud_governance/policy/policy_runners/elasticsearch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py b/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py new file mode 100644 index 000000000..a98e0ada0 --- /dev/null +++ b/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py @@ -0,0 +1,40 @@ +from abc import ABC +from datetime import datetime +from typing import Union + + +from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.policy.policy_runners.common.abstract_upload import AbstractUpload + + +class UploadElasticSearch(AbstractUpload, ABC): + + DEFAULT_UPLOAD_LIMIT = 500 + + def __init__(self): + super().__init__() + self._es_operations = ElasticSearchOperations() + self.__es_host = self._environment_variables_dict.get('es_host', '') + + def upload(self, data: Union[list, dict]): + """ + This method upload data to ElasticSearch + :return: + :rtype: + """ + if self.__es_host: + if self._es_operations.check_elastic_search_connection(): + if data: + if len(data) > self.DEFAULT_UPLOAD_LIMIT: + self._es_operations.upload_data_in_bulk(data_items=data.copy(), index=self._es_index) + else: + for policy_dict in data: + policy_dict['region_name'] = self._region + policy_dict['account'] = self._account + self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index) + logger.info(f'Uploaded the policy results to elasticsearch index: {self._es_index}') + else: + logger.error(f'No data to upload on @{self._account} at {datetime.utcnow()}') + else: + logger.error('ElasticSearch host is not pingable, Please check your connection') diff --git a/docs/source/index.md b/docs/source/index.md index 57f9beba7..06c628fd4 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -11,7 +11,7 @@ This tool support the following policies: * Real time Openshift Cluster cost, User cost * [ec2_idle](../../cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb. -* [ec2_run](../../cloud_governance/policy/aws/ec2_run.py): running ec2. +* [ec2_run](../../cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2. * [ebs_unattached](../../cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status. * [ebs_in_use](../../cloud_governance/policy/aws/ebs_in_use.py): in use volumes. * [tag_resources](../../cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags diff --git a/jenkins/clouds/aws/daily/policies/run_policies.py b/jenkins/clouds/aws/daily/policies/run_policies.py index 36b000fc3..ff1b4afdf 100644 --- a/jenkins/clouds/aws/daily/policies/run_policies.py +++ b/jenkins/clouds/aws/daily/policies/run_policies.py @@ -59,6 +59,7 @@ def get_policies(type: str = None): policies.remove('cost_explorer_payer_billings') policies.remove('spot_savings_analysis') policies.remove('optimize_resources_report') +policies.remove('ec2_run') es_index_env_var = f'-e es_index={ES_INDEX}' if ES_INDEX else '' diff --git a/tests/integration/cloud_governance/aws/tag_non_cluster/test_tag_non_cluster_resources.py b/tests/integration/cloud_governance/aws/tag_non_cluster/test_tag_non_cluster_resources.py index 4b2ff0135..4e7424d1e 100644 --- a/tests/integration/cloud_governance/aws/tag_non_cluster/test_tag_non_cluster_resources.py +++ b/tests/integration/cloud_governance/aws/tag_non_cluster/test_tag_non_cluster_resources.py @@ -35,6 +35,6 @@ def test_tag_non_cluster_update_ec2(): expected_tags = ['User', 'Budget', 'Email', 'Owner', 'Manager', 'Project', 'Environment', 'LaunchTime', 'cg-Name'] instance_tags = ec2_client.describe_tags(Filters=[{'Name': 'resource-id', 'Values': [INSTANCE_ID]}])['Tags'] - actual_tags = [tag.get('Key') for tag in instance_tags] + actual_tags = [tag.get('Key') for tag in instance_tags if tag.get('Key') in expected_tags] assert len(expected_tags) == len(actual_tags) assert expected_tags.sort() == actual_tags.sort() diff --git a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ebs_in_use.py b/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ebs_in_use.py index 29f214c0a..e5f5c20fd 100644 --- a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ebs_in_use.py +++ b/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ebs_in_use.py @@ -4,6 +4,7 @@ from moto import mock_ec2, mock_s3 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations +from cloud_governance.main.environment_variables import environment_variables from cloud_governance.policy.aws.ebs_in_use import EbsInUse os.environ['AWS_DEFAULT_REGION'] = 'us-east-2' @@ -15,7 +16,7 @@ def test_ebs_in_use(): This method test in-use ebs volumes @return: """ - os.environ['policy'] = 'ebs_in_use' + environment_variables.environment_variables_dict['policy'] = 'ebs_in_use' region = os.environ.get('AWS_DEFAULT_REGION', 'us-east-1') ec2_client = boto3.client('ec2', region_name=region) volume_id = ec2_client.create_volume(Size=10, AvailabilityZone='us-east-1a')['VolumeId'] @@ -34,7 +35,7 @@ def test_ebs_in_use_s3_upload(): This method test the data is upload t s3 or not @return: """ - os.environ['policy'] = 'ebs_in_user' + environment_variables.environment_variables_dict['policy'] = 'ebs_in_use' region = os.environ.get('AWS_DEFAULT_REGION', 'us-east-1') ec2_client = boto3.client('ec2', region_name=region) default_ami_id = 'ami-03cf127a' diff --git a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ec2_idle.py b/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ec2_idle.py index e0938f723..fafc79203 100644 --- a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ec2_idle.py +++ b/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ec2_idle.py @@ -3,6 +3,7 @@ import boto3 from moto import mock_ec2, mock_cloudwatch +from cloud_governance.main.environment_variables import environment_variables from cloud_governance.policy.aws.ec2_idle import EC2Idle AWS_DEFAULT_REGION = 'us-east-2' @@ -36,6 +37,8 @@ def test_ec2_idle(): This method check the instance is deleted or not @return: """ + environment_variables.environment_variables_dict['policy'] = 'ec2_idle' + environment_variables.environment_variables_dict['dry_run'] = 'no' expected_result = 'stopped' ec2_client = boto3.client('ec2', region_name=AWS_DEFAULT_REGION) default_ami_id = 'ami-03cf127a' diff --git a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ec2_run.py b/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ec2_run.py deleted file mode 100644 index cd7b2c6a6..000000000 --- a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ec2_run.py +++ /dev/null @@ -1,47 +0,0 @@ - -import boto3 -from moto import mock_ec2, mock_s3 - -from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations -from cloud_governance.policy.aws.ec2_run import EC2Run - -AWS_DEFAULT_REGION = 'us-east-2' - - -@mock_ec2 -def test_ec2_run(): - """ - This method test running ec2 instances - @return: - """ - region = AWS_DEFAULT_REGION - ec2_client = boto3.client('ec2', region_name=region) - default_ami_id = 'ami-03cf127a' - ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1) - ec2_run = EC2Run() - ec2_run.set_dryrun(value='yes') - ec2_run.set_policy(value='ec2_run') - ec2_run.set_region(value=AWS_DEFAULT_REGION) - assert 1 == len(ec2_run.run()) - - -@mock_s3 -@mock_ec2 -def test_ec2_run_s3_upload(): - """ - This method test the data is upload t s3 or not - @return: - """ - region = AWS_DEFAULT_REGION - ec2_client = boto3.client('ec2', region_name=region) - default_ami_id = 'ami-03cf127a' - ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1) - s3_client = boto3.client('s3', region_name='us-east-1') - s3_client.create_bucket(Bucket='test-upload-data', CreateBucketConfiguration={'LocationConstraint': 'us-east-2'}) - policy_output = 's3://test-upload-data/test' - s3operations = S3Operations(region_name='us-east-1') - ec2_run = EC2Run() - ec2_run.set_dryrun(value='yes') - ec2_run.set_policy(value='ec2_run') - ec2_run.set_region(value=AWS_DEFAULT_REGION) - assert s3operations.save_results_to_s3(policy='ec2_run', policy_output=policy_output, policy_result=ec2_run.run()) is None diff --git a/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py b/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py index 0bd1aef7e..ae962bc92 100644 --- a/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py +++ b/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py @@ -1,3 +1,5 @@ +import datetime + import boto3 import tempfile import os @@ -6,9 +8,14 @@ from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations # walk around for moto DeprecationWarning import warnings + +from cloud_governance.main.aws_main_operations import AWSMainOperations +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.aws.cleanup.ec2_run import EC2Run + with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) - from moto import mock_s3 + from moto import mock_s3, mock_iam, mock_ec2 @mock_s3 @@ -174,11 +181,57 @@ def test_folder_delete(): assert not s3operations.file_exist(bucket='ais-server', key='test-data', file_name=expected_files_list[1]) +@mock_iam +@mock_ec2 +@mock_s3 def test_get_s3_latest_policy_file(): - s3_operations = S3Operations(region_name='us-east-1', bucket='redhat-cloud-governance', logs_bucket_key='logs') - assert s3_operations._S3Operations__get_s3_latest_policy_file(policy='ec2-idle') - - + region_name = 'us-east-1' + bucket_name = 'test_s3_bucket' + s3_resource = boto3.resource('s3', region_name=region_name) + s3_resource.create_bucket(Bucket=bucket_name) + environment_variables.environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AWS' + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'yes' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name + environment_variables.environment_variables_dict['policy_output'] = f's3://{bucket_name}/tests' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + aws_main_operations = AWSMainOperations() + aws_main_operations.run() + current_date = datetime.datetime.now().date().__str__().replace('-', '/') + prefix = f'tests/{region_name}/ec2-run/{current_date}' + s3_operations = S3Operations(region_name=region_name, bucket=bucket_name, logs_bucket_key='tests') + assert s3_operations._S3Operations__get_s3_latest_policy_file(policy='ec2-run', key_prefix=prefix) + + +@mock_iam +@mock_ec2 +@mock_s3 def test_get_last_s3_policy_content(): - s3_operations = S3Operations(region_name='us-east-1', bucket='redhat-cloud-governance', logs_bucket_key='logs') - assert s3_operations.get_last_s3_policy_content(policy='ec2-idle', file_name='resources.json') + region_name = 'us-east-1' + bucket_name = 'test_s3_bucket' + s3_resource = boto3.resource('s3', region_name=region_name) + s3_resource.create_bucket(Bucket=bucket_name) + environment_variables.environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AWS' + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'yes' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name + environment_variables.environment_variables_dict['policy_output'] = f's3://{bucket_name}/tests' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] + ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + aws_main_operations = AWSMainOperations() + aws_main_operations.run() + current_date = datetime.datetime.now().date().__str__().replace('-', '/') + key_prefix = f'tests/{region_name}/ec2-run/{current_date}' + s3_operations = S3Operations(region_name='us-east-1', bucket=bucket_name, logs_bucket_key='tests') + assert s3_operations.get_last_s3_policy_content(policy='ec2-run', file_name='resources.json', key_prefix=key_prefix) diff --git a/tests/unittest/cloud_governance/common/helpers/__init__.py b/tests/unittest/cloud_governance/common/helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unittest/cloud_governance/common/helpers/aws/__init__.py b/tests/unittest/cloud_governance/common/helpers/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py b/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py new file mode 100644 index 000000000..b6329f6ed --- /dev/null +++ b/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py @@ -0,0 +1,215 @@ +import datetime + +import boto3 +from moto import mock_ec2, mock_s3, mock_iam + +from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations +from cloud_governance.main.environment_variables import environment_variables + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_get_tag_name_from_tags(): + """ + This method tests get_tag_name_from_tags method + :return: + :rtype: + """ + aws_cleanup_operations = AWSCleanUpOperations() + tags = [{'Key': "Name", "Value": "Unittest"}] + tag_value = aws_cleanup_operations.get_tag_name_from_tags(tags=tags, tag_name="Name") + assert tag_value == "Unittest" + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_get_clean_up_days_count(): + """ + This method tests get_clean_up_days_count method + :return: + :rtype: + """ + environment_variables.environment_variables_dict['dry_run'] = 'yes' + aws_cleanup_operations = AWSCleanUpOperations() + tags = [{'Key': "Name", "Value": "Unittest"}] + days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + assert days_count == 1 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_get_clean_up_days_count_already_exists(): + """ + This method tests get_clean_up_days_count method + :return: + :rtype: + """ + environment_variables.environment_variables_dict['dry_run'] = 'yes' + aws_cleanup_operations = AWSCleanUpOperations() + mock_date = (datetime.datetime.now() - datetime.timedelta(days=1)).date() + tags = [{'Key': "Name", "Value": "Unittest"}, {'Key': "DaysCount", "Value": f'{mock_date}@1'}] + days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + assert days_count == 1 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_get_clean_up_days_count_already_updated_today(): + """ + This method tests get_clean_up_days_count method + :return: + :rtype: + """ + environment_variables.environment_variables_dict['dry_run'] = 'yes' + aws_cleanup_operations = AWSCleanUpOperations() + mock_date = str(datetime.datetime.now().date()) + tags = [{'Key': "Name", "Value": "Unittest"}, {'Key': "DaysCount", "Value": f'{mock_date}@1'}] + days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + assert days_count == 2 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_get_skip_policy_value_policy_tag(): + """ + This method tests get_skip_policy_value + :return: + :rtype: + """ + aws_cleanup_operations = AWSCleanUpOperations() + tags = [{'Key': "Name", "Value": "Unittest"}, + {'Key': "Policy", "Value": "NotDelete"}] + tag_value = aws_cleanup_operations.get_skip_policy_value(tags=tags) + assert tag_value == "NotDelete".upper() + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_get_skip_policy_value_skip_tag(): + """ + This method tests get_skip_policy_value + :return: + :rtype: + """ + aws_cleanup_operations = AWSCleanUpOperations() + tags = [{'Key': "Name", "Value": "Unittest"}, + {'Key': "Skip", "Value": "NotDelete"}] + tag_value = aws_cleanup_operations.get_skip_policy_value(tags=tags) + assert tag_value == "NotDelete".upper() + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_resource(): + """ + This method tests _delete_resource + :return: + :rtype: + """ + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + tags = [{'Key': 'User', 'Value': 'cloud-governance'}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) + if resource: + resource_id = resource[0].get('InstanceId') + aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations._delete_resource(resource_id=resource_id) + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_update_resource_day_count_tag(): + """ + This method tests update_resource_day_count_tag + :return: + :rtype: + """ + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + if resource: + resource_id = resource[0].get('InstanceId') + aws_cleanup_operations = AWSCleanUpOperations() + cleanup_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + aws_cleanup_operations.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags) + instances = ec2_client.describe_instances()['Reservations'] + tag_value = aws_cleanup_operations.get_tag_name_from_tags(instances[0]['Instances'][0].get('Tags'), tag_name='DaysCount') + assert tag_value == str(datetime.datetime.utcnow().date()) + "@0" + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_update_resource_day_count_tag_exists_tag(): + """ + This method tests update_resource_tags + :return: + :rtype: + """ + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + environment_variables.environment_variables_dict['dry_run'] = 'no' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DaysCount", "Value": f'{mock_date}@1'}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + if resource: + resource_id = resource[0].get('InstanceId') + aws_cleanup_operations = AWSCleanUpOperations() + cleanup_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + aws_cleanup_operations.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags) + instances = ec2_client.describe_instances()['Reservations'] + tag_value = aws_cleanup_operations.get_tag_name_from_tags(instances[0]['Instances'][0].get('Tags'), tag_name='DaysCount') + assert tag_value == str(datetime.datetime.utcnow().date()) + "@2" + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_update_resource_day_count_tag_updated_tag_today(): + """ + This method tests update_resource_tags + :return: + :rtype: + """ + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + environment_variables.environment_variables_dict['dry_run'] = 'no' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = datetime.datetime.utcnow().date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DryRunYesDays", "Value": f'{mock_date}@1'}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + if resource: + resource_id = resource[0].get('InstanceId') + aws_cleanup_operations = AWSCleanUpOperations() + cleanup_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + aws_cleanup_operations.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags) + instances = ec2_client.describe_instances()['Reservations'] + tag_value = aws_cleanup_operations.get_tag_name_from_tags(instances[0]['Instances'][0].get('Tags'), tag_name='DaysCount') + assert tag_value == str(datetime.datetime.utcnow().date()) + "@1" + diff --git a/tests/unittest/cloud_governance/common/helpers/test_cleanup_operations.py b/tests/unittest/cloud_governance/common/helpers/test_cleanup_operations.py new file mode 100644 index 000000000..7696cb5e1 --- /dev/null +++ b/tests/unittest/cloud_governance/common/helpers/test_cleanup_operations.py @@ -0,0 +1,92 @@ +import datetime + +import boto3 +from moto import mock_ec2, mock_s3, mock_iam + +from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations +from cloud_governance.main.environment_variables import environment_variables + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_verify_and_delete_resource_not_stopped(): + """ + This method tests verify_and_delete_resource + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + resource_id = resource[0].get('InstanceId') + aws_cleanup_operations = AWSCleanUpOperations() + clean_up_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + assert aws_cleanup_operations.verify_and_delete_resource(resource_id=resource_id, tags=tags, + clean_up_days=clean_up_days) + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_verify_and_delete_resource_stopped(): + """ + This method tests verify_and_delete_resource + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.datetime.now() - datetime.timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DryRunNoDays", "Value": f"{mock_date}@3"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + resource_id = resource[0].get('InstanceId') + aws_cleanup_operations = AWSCleanUpOperations() + clean_up_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + assert aws_cleanup_operations.verify_and_delete_resource(resource_id=resource_id, tags=tags, + clean_up_days=clean_up_days) + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_verify_and_delete_resource_skip(): + """ + This method tests verify_and_delete_resource + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'yes' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.datetime.now() - datetime.timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DryRunNoDays", "Value": f"{mock_date}@3"}, + {'Key': "Skip", "Value": f"notdelete"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + resource_id = resource[0].get('InstanceId') + aws_cleanup_operations = AWSCleanUpOperations() + clean_up_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) + assert aws_cleanup_operations.verify_and_delete_resource(resource_id=resource_id, tags=tags, + clean_up_days=clean_up_days) + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 diff --git a/tests/unittest/cloud_governance/common/helpers/test_json_datetime_encoder.py b/tests/unittest/cloud_governance/common/helpers/test_json_datetime_encoder.py new file mode 100644 index 000000000..1dc3c17c5 --- /dev/null +++ b/tests/unittest/cloud_governance/common/helpers/test_json_datetime_encoder.py @@ -0,0 +1,11 @@ +import datetime +import json + +from cloud_governance.common.helpers.json_datetime_encoder import JsonDateTimeEncoder + + +def test_json_datetime_encoder(): + current_date = datetime.datetime.now() + data_list = [current_date, "Unittest"] + json_data = json.dumps(data_list, cls=JsonDateTimeEncoder) + assert json_data == json.dumps([str(current_date.isoformat()), "Unittest"]) diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/__init__.py b/tests/unittest/cloud_governance/policy/aws/cleanup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py new file mode 100644 index 000000000..1f485956f --- /dev/null +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py @@ -0,0 +1,365 @@ +from datetime import datetime, timedelta + +import boto3 +from moto import mock_ec2, mock_s3, mock_iam + +from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.aws.cleanup.ec2_run import EC2Run + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_ec2_run(): + """ + This method tests ec2_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', + []) + ec2_run = EC2Run() + running_instances_data = ec2_run.run() + assert running_instances_data[0].get('ResourceStopped') == 'False' + assert running_instances_data[0].get('InstanceState') == 'running' + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_ec2_run_alert(): + """ + This method tests ec2_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get( + 'Instances', + []) + resource = resource[0] + ec2_run = EC2Run() + running_instances_data = ec2_run.run() + assert running_instances_data == [ + { + 'ResourceId': resource.get('InstanceId'), + 'User': 'cloud-governance', + 'SkipPolicy': 'NA', + 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'InstanceType': resource.get('InstanceType'), + 'InstanceState': 'running', + 'StateTransitionReason': resource.get('StateTransitionReason'), + 'RunningDays': 0, + 'CleanUpDays': 1, + 'DryRun': 'no', + 'Name': 'Unittest', + 'RegionName': 'ap-south-1', + 'ResourceStopped': 'False' + } + ] + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 + + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_ec2_run_alert_stopped(): + """ + This method tests ec2_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get( + 'Instances', + []) + resource = resource[0] + ec2_run = EC2Run() + running_instances_data = ec2_run.run() + assert running_instances_data == [ + { + 'ResourceId': resource.get('InstanceId'), + 'User': 'cloud-governance', + 'SkipPolicy': 'NA', + 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'InstanceType': resource.get('InstanceType'), + 'InstanceState': 'stopped', + 'StateTransitionReason': resource.get('StateTransitionReason'), + 'RunningDays': 0, + 'CleanUpDays': 4, + 'DryRun': 'no', + 'Name': 'Unittest', + 'RegionName': 'ap-south-1', + 'ResourceStopped': 'True' + } + ] + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_ec2_run_alert_skip(): + """ + This method tests ec2_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}, {'Key': "Skip", "Value": f"notdelete"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get( + 'Instances', + []) + resource = resource[0] + ec2_run = EC2Run() + running_instances_data = ec2_run.run() + assert running_instances_data == [ + { + 'ResourceId': resource.get('InstanceId'), + 'User': 'cloud-governance', + 'SkipPolicy': 'NOTDELETE', + 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'InstanceType': resource.get('InstanceType'), + 'InstanceState': 'running', + 'StateTransitionReason': resource.get('StateTransitionReason'), + 'RunningDays': 0, + 'CleanUpDays': 4, + 'DryRun': 'no', + 'Name': 'Unittest', + 'RegionName': 'ap-south-1', + 'ResourceStopped': 'False' + } + ] + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_ec2_run_stop_reset(): + """ + This method tests ec2_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get( + 'Instances', + []) + resource = resource[0] + ec2_run = EC2Run() + running_instances_data = ec2_run.run() + assert running_instances_data == [ + { + 'ResourceId': resource.get('InstanceId'), + 'User': 'cloud-governance', + 'SkipPolicy': 'NA', + 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'InstanceType': resource.get('InstanceType'), + 'InstanceState': 'stopped', + 'StateTransitionReason': resource.get('StateTransitionReason'), + 'RunningDays': 0, + 'CleanUpDays': 4, + 'DryRun': 'no', + 'Name': 'Unittest', + 'RegionName': 'ap-south-1', + 'ResourceStopped': 'True' + } + ] + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 + ec2_run.run() + instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations'] + instances = instances[0]['Instances'][0] + aws_cleanup_operations = AWSCleanUpOperations() + assert aws_cleanup_operations.get_tag_name_from_tags(tags=instances.get('Tags'), tag_name='DaysCount').split('@')[-1] == '0' + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_ec2_run_stop_start(): + """ + This method tests ec2_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get( + 'Instances', + []) + resource = resource[0] + ec2_run = EC2Run() + running_instances_data = ec2_run.run() + assert running_instances_data == [ + { + 'ResourceId': resource.get('InstanceId'), + 'User': 'cloud-governance', + 'SkipPolicy': 'NA', + 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'InstanceType': resource.get('InstanceType'), + 'InstanceState': 'stopped', + 'StateTransitionReason': resource.get('StateTransitionReason'), + 'RunningDays': 0, + 'CleanUpDays': 4, + 'DryRun': 'no', + 'Name': 'Unittest', + 'RegionName': 'ap-south-1', + 'ResourceStopped': 'True' + } + ] + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 + ec2_run.run() + instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations'] + instances = instances[0]['Instances'][0] + aws_cleanup_operations = AWSCleanUpOperations() + assert aws_cleanup_operations.get_tag_name_from_tags(tags=instances.get('Tags'), tag_name='DaysCount').split('@')[-1] == '0' + ec2_client.start_instances(InstanceIds=[resource.get('InstanceId')]) + ec2_run.run() + instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations'] + instances = instances[0]['Instances'][0] + assert aws_cleanup_operations.get_tag_name_from_tags(tags=instances.get('Tags'), tag_name='DaysCount').split('@')[-1] == '1' + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_ec2_force_delete(): + """ + This method tests ec2_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['FORCE_DELETE'] = True + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get( + 'Instances', + []) + resource = resource[0] + environment_variables.environment_variables_dict['RESOURCE_ID'] = resource.get('InstanceId') + ec2_run = EC2Run() + running_instances_data = ec2_run.run() + assert running_instances_data == [ + { + 'ResourceId': resource.get('InstanceId'), + 'User': 'cloud-governance', + 'SkipPolicy': 'NA', + 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'InstanceType': resource.get('InstanceType'), + 'InstanceState': 'stopped', + 'StateTransitionReason': resource.get('StateTransitionReason'), + 'RunningDays': 0, + 'CleanUpDays': 4, + 'DryRun': 'no', + 'Name': 'Unittest', + 'RegionName': 'ap-south-1', + 'ResourceStopped': 'True', + 'ForceDeleted': 'True' + } + ] + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_ec2_force_delete_skip(): + """ + This method tests ec2_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['FORCE_DELETE'] = True + environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['dry_run'] = 'yes' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' + ec2_client = boto3.client('ec2', region_name='ap-south-1') + default_ami_id = 'ami-03cf127a' + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}] + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get( + 'Instances', + []) + resource = resource[0] + environment_variables.environment_variables_dict['RESOURCE_ID'] = resource.get('InstanceId') + ec2_run = EC2Run() + running_instances_data = ec2_run.run() + assert running_instances_data == [ + { + 'ResourceId': resource.get('InstanceId'), + 'User': 'cloud-governance', + 'SkipPolicy': 'NA', + 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + 'InstanceType': resource.get('InstanceType'), + 'InstanceState': 'running', + 'StateTransitionReason': resource.get('StateTransitionReason'), + 'RunningDays': 0, + 'CleanUpDays': 4, + 'DryRun': 'yes', + 'Name': 'Unittest', + 'RegionName': 'ap-south-1', + 'ResourceStopped': 'False', + } + ] + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 diff --git a/tests/unittest/cloud_governance/policy/policy_runners/__init__.py b/tests/unittest/cloud_governance/policy/policy_runners/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unittest/cloud_governance/policy/policy_runners/aws/__init__.py b/tests/unittest/cloud_governance/policy/policy_runners/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unittest/cloud_governance/policy/policy_runners/aws/test_upload_s3.py b/tests/unittest/cloud_governance/policy/policy_runners/aws/test_upload_s3.py new file mode 100644 index 000000000..60109efaa --- /dev/null +++ b/tests/unittest/cloud_governance/policy/policy_runners/aws/test_upload_s3.py @@ -0,0 +1,53 @@ +import datetime + +import boto3 +from moto import mock_s3 + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.policy_runners.aws.upload_s3 import UploadS3 + + +@mock_s3 +def test_upload_s3_list(): + """ + This method tests the data to be uploaded to s3 + :return: + :rtype: + """ + bucket_name = "cloud_governance_unitest" + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': 'ap-south-1'}) + environment_variables.environment_variables_dict["policy_output"] = f"s3://{bucket_name}/tests" + + data = [ + datetime.datetime.now(), "unitest", "cloud_governance" + ] + upload_s3 = UploadS3() + upload_s3.upload(data=data) + assert len(s3_client.list_objects_v2(Bucket=bucket_name).get('Contents')) == 1 + + +@mock_s3 +def test_upload_s3_dict(): + """ + This method tests the data to be uploaded to s3 + :return: + :rtype: + """ + bucket_name = "cloud_governance_unitest" + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': 'ap-south-1'}) + environment_variables.environment_variables_dict["policy_output"] = f"s3://{bucket_name}/tests" + + data = { + "timestamp": datetime.datetime.now(), + "policy": "test_unitest", + "runner": "cloud_governance", + "results": [ + {"date": datetime.datetime.now()}, + {"ResourceId": bucket_name, "CreateDate": datetime.datetime.now().date()} + ] + } + upload_s3 = UploadS3() + upload_s3.upload(data=data) + assert len(s3_client.list_objects_v2(Bucket=bucket_name).get('Contents')) == 1