From b9b55afed9b69803794a2cbdac60b09ea051ef5d Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Tue, 28 May 2024 22:41:21 +0530 Subject: [PATCH] Refractored the S3 inactive to new approach --- .../common/clouds/aws/s3/s3_operations.py | 62 +++++++ .../main/main_oerations/main_operations.py | 3 +- cloud_governance/policy/aws/s3_inactive.py | 78 ++++---- .../helpers/aws/aws_policy_operations.py | 19 +- .../zombie_non_cluster/test_empty_buckets.py | 66 ------- .../policy/aws/test_s3_inactive.py | 167 ++++++++++++++++++ 6 files changed, 288 insertions(+), 107 deletions(-) delete mode 100644 tests/unittest/cloud_governance/aws/zombie_non_cluster/test_empty_buckets.py create mode 100644 tests/unittest/cloud_governance/policy/aws/test_s3_inactive.py diff --git a/cloud_governance/common/clouds/aws/s3/s3_operations.py b/cloud_governance/common/clouds/aws/s3/s3_operations.py index d3a5ef14e..f00ddeb4c 100644 --- a/cloud_governance/common/clouds/aws/s3/s3_operations.py +++ b/cloud_governance/common/clouds/aws/s3/s3_operations.py @@ -9,6 +9,7 @@ from os import listdir from os.path import isfile, join +from cloud_governance.common.logger.init_logger import logger from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp @@ -325,3 +326,64 @@ def get_last_s3_policy_content(self, policy: str = '', file_name: str = '', s3_f os.system(f"gzip -d {local_file}") with open(os.path.join(temp_local_directory, file_name)) as f: return f.read() + + def list_buckets(self): + """ + This method list all buckets + :return: + """ + try: + return self.__s3_client.list_buckets().get('Buckets', []) + except Exception as err: + return [] + + def get_bucket_tagging(self, bucket_name: str, **kwargs): + """ + This method get tagging buckets + :param bucket_name: + :return: + """ + tags = [] + try: + bucket_tags = self.__s3_client.get_bucket_tagging(Bucket=bucket_name, **kwargs) + tags = bucket_tags.get('TagSet', []) + except Exception as err: + logger.error(err) + return tags + + def list_objects_v2(self, bucket_name: str, prefix: str = '', **kwargs): + """ + This method list all objects of a bucket + :param bucket_name: + :param prefix: + :return: + """ + bucket_data = {} + try: + bucket_data = self.__s3_client.list_objects_v2(Bucket=bucket_name, Prefix=prefix, **kwargs) + except Exception as err: + logger.error(err) + return bucket_data + + def get_bucket_contents(self, bucket_name: str, **kwargs): + """ + This method get contents of a bucket + :param bucket_name: + :param kwargs: + :return: + """ + bucket_data = self.list_objects_v2(bucket_name, **kwargs) + return bucket_data.get('Contents', []) + + def get_bucket_location(self, bucket_name: str, **kwargs): + """ + This method get location of a bucket + :param bucket_name: + :param kwargs: + :return: + """ + try: + return self.__s3_client.get_bucket_location(Bucket=bucket_name, **kwargs).get('LocationConstraint', self.__region) + except Exception as err: + logger.error(err) + return self.__region diff --git a/cloud_governance/main/main_oerations/main_operations.py b/cloud_governance/main/main_oerations/main_operations.py index 1a43891bb..fb38fce5b 100644 --- a/cloud_governance/main/main_oerations/main_operations.py +++ b/cloud_governance/main/main_oerations/main_operations.py @@ -39,7 +39,8 @@ def run(self): for policy_type, policies in policies_list.items(): # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement if self._policy in policies and self._policy in ["instance_run", "unattached_volume", "cluster_run", - "ip_unattached", "unused_nat_gateway", "instance_idle"]: + "ip_unattached", "unused_nat_gateway", "instance_idle", + "s3_inactive"]: source = policy_type if Utils.equal_ignore_case(policy_type, self._public_cloud_name): source = '' diff --git a/cloud_governance/policy/aws/s3_inactive.py b/cloud_governance/policy/aws/s3_inactive.py index b0ae5eb09..f40a8e930 100644 --- a/cloud_governance/policy/aws/s3_inactive.py +++ b/cloud_governance/policy/aws/s3_inactive.py @@ -2,57 +2,57 @@ from botocore.exceptions import ClientError from cloud_governance.common.logger.init_logger import logger +from cloud_governance.common.utils.utils import Utils +from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy -class S3Inactive(NonClusterZombiePolicy): +class S3Inactive(AWSPolicyOperations): """ This class sends an alert mail for empty bucket to the user after 4 days and delete after 7 days. """ + RESOURCE_ACTION = 'Delete' + def __init__(self): super().__init__() + self.__global_active_cluster_ids = self._get_global_active_cluster_ids() - def run(self): - """ - This method returns all Empty buckets and delete if dry_run no - @return: + def run_policy_operations(self): """ - return self.__delete_s3_inactive() - - def __delete_s3_inactive(self): - """ - This method delete the empty bucket more than 7 days - @return: + This method returns all Empty buckets + :return: + :rtype: """ empty_buckets = [] - buckets = self._s3_client.list_buckets()['Buckets'] - for bucket in buckets: - bucket_empty = False - empty_days = 0 + s3_buckets = self._s3operations.list_buckets() + for bucket in s3_buckets: bucket_name = bucket.get('Name') - try: - try: - bucket_tags = self._s3_client.get_bucket_tagging(Bucket=bucket_name) - tags = bucket_tags.get('TagSet') - except ClientError: - tags = [] - bucket_data = self._s3_client.list_objects_v2(Bucket=bucket_name) - if not bucket_data.get('Contents'): - if not self._check_cluster_tag(tags=tags): - if not self._get_tag_name_from_tags(tags=tags, tag_name='Name'): - tags.append({'Key': 'Name', 'Value': bucket_name}) - empty_days = self._get_resource_last_used_days(tags=tags) - bucket_empty = True - if not self._get_tag_name_from_tags(tags=tags, tag_name='User'): - region = self._s3_client.get_bucket_location(Bucket=bucket_name)['LocationConstraint'] - self._cloudtrail.set_cloudtrail(region_name=region) - empty_bucket = self._check_resource_and_delete(resource_name='S3 Bucket', resource_id='Name', resource_type='CreateBucket', resource=bucket, empty_days=empty_days, days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE, tags=tags) - if empty_bucket: - empty_buckets.append({'ResourceId': bucket.get('Name'), 'Name': bucket.get('Name'), 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'), 'Date': str(bucket.get('CreationDate')), 'Days': str(empty_days), 'Skip': self._get_policy_value(tags=tags)}) - else: - empty_days = 0 - self._update_resource_tags(resource_id=bucket_name, tags=tags, left_out_days=empty_days, resource_left_out=bucket_empty) - except Exception as err: - logger.info(f'{err}, {bucket.get("Name")}') + tags = self._s3operations.get_bucket_tagging(bucket_name) + cleanup_result = False + cluster_tag = self._get_cluster_tag(tags=tags) + cleanup_days = 0 + s3_contents = self._s3operations.get_bucket_contents(bucket_name=bucket_name) + if cluster_tag not in self.__global_active_cluster_ids and len(s3_contents) == 0: + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource(resource_id=bucket_name, tags=tags, + clean_up_days=cleanup_days) + region = self._s3operations.get_bucket_location(bucket_name=bucket_name) + resource_data = self._get_es_schema(resource_id=bucket_name, + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + skip_policy=self.get_skip_policy_value(tags=tags), + cleanup_days=cleanup_days, + dry_run=self._dry_run, + name=bucket_name, + region=region, + cleanup_result=str(cleanup_result), + resource_action=self.RESOURCE_ACTION, + cloud_name=self._cloud_name, + resource_type='EmptyBucket', + resource_state="Empty", + unit_price=0) + empty_buckets.append(resource_data) + if not cleanup_result: + self.update_resource_day_count_tag(resource_id=bucket_name, cleanup_days=cleanup_days, tags=tags) + return empty_buckets diff --git a/cloud_governance/policy/helpers/aws/aws_policy_operations.py b/cloud_governance/policy/helpers/aws/aws_policy_operations.py index 4c67f81c3..ffa0d66a6 100644 --- a/cloud_governance/policy/helpers/aws/aws_policy_operations.py +++ b/cloud_governance/policy/helpers/aws/aws_policy_operations.py @@ -18,7 +18,7 @@ def __init__(self): super().__init__() self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') self._cloud_name = 'AWS' - self.__s3operations = S3Operations(region_name=self._region) + self._s3operations = S3Operations(region_name=self._region) self._ec2_client = boto3.client('ec2', region_name=self._region) self._ec2_operations = EC2Operations(region=self._region) self._cloudwatch = CloudWatchOperations(region=self._region) @@ -173,6 +173,23 @@ def _get_active_cluster_ids(self): break return cluster_ids + def _get_global_active_cluster_ids(self): + """ + This method returns the global active cluster ids + :return: + """ + cluster_ids = [] + active_regions = self._ec2_operations.get_active_regions() + for region in active_regions: + active_instances = self._ec2_operations.get_ec2_instance_list(ec2_client=boto3.client('ec2', + region_name=region)) + for instance in active_instances: + for tag in instance.get('Tags', []): + if tag.get('Key', '').startswith('kubernetes.io/cluster'): + cluster_ids.append(tag.get('Key')) + break + return cluster_ids + def _get_cluster_tag(self, tags: list): """ This method returns the cluster_tag diff --git a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_empty_buckets.py b/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_empty_buckets.py deleted file mode 100644 index 9b420d1c0..000000000 --- a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_empty_buckets.py +++ /dev/null @@ -1,66 +0,0 @@ -import os - -import boto3 -from moto import mock_s3, mock_ec2 - -from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy - -os.environ['AWS_DEFAULT_REGION'] = 'us-east-2' -os.environ['dry_run'] = 'no' - - -@mock_ec2 -@mock_s3 -def test_s3_inactive(): - """ - This method tests delete of empty buckets - @return: - """ - os.environ['policy'] = 's3_inactive' - s3_client = boto3.client('s3', region_name='us-east-1') - s3_client.create_bucket(Bucket='cloud-governance-test-s3-empty-delete', - CreateBucketConfiguration={'LocationConstraint': 'us-east-2'}) - s3_inactive = NonClusterZombiePolicy() - s3_inactive.set_dryrun(value='no') - s3_inactive.set_policy(value='s3_inactive') - s3_inactive.DAYS_TO_TRIGGER_RESOURCE_MAIL = -1 - s3_inactive._check_resource_and_delete(resource_name='S3 Bucket', - resource_id='Name', - resource_type='CreateBucket', - resource=s3_client.list_buckets()['Buckets'][0], - empty_days=0, - days_to_delete_resource=0) - buckets = s3_client.list_buckets()['Buckets'] - assert len(buckets) == 0 - - -@mock_ec2 -@mock_s3 -def test_s3_inactive_not_delete(): - """ - This method tests not delete of empty buckets, if policy=NOT_DELETE - @return: - """ - os.environ['policy'] = 's3_inactive' - tags = [ - {'Key': 'Name', 'Value': 'CloudGovernanceTestEmptyBucket'}, - {'Key': 'Owner', 'Value': 'CloudGovernance'}, - {'Key': 'policy', 'Value': 'notdelete'} - ] - s3_client = boto3.client('s3', region_name='us-east-1') - s3_client.create_bucket(Bucket='cloud-governance-test-s3-empty-delete', - CreateBucketConfiguration={'LocationConstraint': 'us-east-2'}) - s3_client.put_bucket_tagging(Bucket='cloud-governance-test-s3-empty-delete', Tagging={'TagSet': tags}) - s3_inactive = NonClusterZombiePolicy() - s3_inactive.set_dryrun(value='no') - s3_inactive.set_policy(value='s3_inactive') - s3_inactive.DAYS_TO_TRIGGER_RESOURCE_MAIL = -1 - s3_inactive._check_resource_and_delete(resource_name='S3 Bucket', - resource_id='Name', - resource_type='CreateBucket', - resource=s3_client.list_buckets()['Buckets'][0], - empty_days=0, - days_to_delete_resource=0, - tags=tags) - buckets = s3_client.list_buckets()['Buckets'] - assert len(buckets) == 1 diff --git a/tests/unittest/cloud_governance/policy/aws/test_s3_inactive.py b/tests/unittest/cloud_governance/policy/aws/test_s3_inactive.py new file mode 100644 index 000000000..f1bd2cdc0 --- /dev/null +++ b/tests/unittest/cloud_governance/policy/aws/test_s3_inactive.py @@ -0,0 +1,167 @@ +import tempfile + +import boto3 +from moto import mock_s3, mock_ec2 + +from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations +from cloud_governance.common.clouds.aws.utils.common_methods import get_tag_value_from_tags +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.aws.s3_inactive import S3Inactive +from tests.unittest.configs import DRY_RUN_YES, AWS_DEFAULT_REGION, TEST_USER_NAME, CURRENT_DATE, DRY_RUN_NO, \ + DEFAULT_AMI_ID, INSTANCE_TYPE_T2_MICRO + + +@mock_ec2 +@mock_s3 +def test_s3_inactive(): + """ + This method tests lists empty buckets + @return: + """ + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['policy'] = 's3_inactive' + tags = [{'Key': 'User', 'Value': TEST_USER_NAME}] + s3_client = boto3.client('s3', region_name=AWS_DEFAULT_REGION) + s3_client.create_bucket(Bucket=TEST_USER_NAME, CreateBucketConfiguration={'LocationConstraint': AWS_DEFAULT_REGION}) + s3_client.put_bucket_tagging(Bucket=TEST_USER_NAME, Tagging={'TagSet': tags}) + s3_operations = S3Operations(region_name=AWS_DEFAULT_REGION) + # run s3_inactive + s3_inactive = S3Inactive() + response = s3_inactive.run() + assert len(s3_operations.list_buckets()) == 1 + assert len(response) == 1 + assert response[0]['CleanUpDays'] == 1 + assert get_tag_value_from_tags(tags=s3_operations.get_bucket_tagging(bucket_name=TEST_USER_NAME), + tag_name='DaysCount') == f"{CURRENT_DATE}@1" + + +@mock_ec2 +@mock_s3 +def test_s3_inactive_dry_run_yes(): + """ + This method tests collects empty buckets on dry_run=yes + @return: + """ + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['policy'] = 's3_inactive' + tags = [{'Key': 'User', 'Value': TEST_USER_NAME}, {'Key': 'DaysCount', 'Value': f'{CURRENT_DATE}@7'}] + s3_client = boto3.client('s3', region_name=AWS_DEFAULT_REGION) + s3_client.create_bucket(Bucket=TEST_USER_NAME, CreateBucketConfiguration={'LocationConstraint': AWS_DEFAULT_REGION}) + s3_client.put_bucket_tagging(Bucket=TEST_USER_NAME, Tagging={'TagSet': tags}) + s3_operations = S3Operations(region_name=AWS_DEFAULT_REGION) + # run s3_inactive + s3_inactive = S3Inactive() + response = s3_inactive.run() + assert len(s3_operations.list_buckets()) == 1 + assert len(response) == 1 + assert response[0]['CleanUpDays'] == 0 + assert get_tag_value_from_tags(tags=s3_operations.get_bucket_tagging(bucket_name=TEST_USER_NAME), + tag_name='DaysCount') == f"{CURRENT_DATE}@0" + + +@mock_ec2 +@mock_s3 +def test_s3_inactive_delete(): + """ + This method tests delete s3 empty bucket + @return: + """ + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['policy'] = 's3_inactive' + tags = [{'Key': 'User', 'Value': TEST_USER_NAME}, {'Key': 'DaysCount', 'Value': f'{CURRENT_DATE}@7'}] + s3_client = boto3.client('s3', region_name=AWS_DEFAULT_REGION) + s3_client.create_bucket(Bucket=TEST_USER_NAME, CreateBucketConfiguration={'LocationConstraint': AWS_DEFAULT_REGION}) + s3_client.put_bucket_tagging(Bucket=TEST_USER_NAME, Tagging={'TagSet': tags}) + s3_operations = S3Operations(region_name=AWS_DEFAULT_REGION) + # run s3_inactive + s3_inactive = S3Inactive() + response = s3_inactive.run() + assert len(s3_operations.list_buckets()) == 0 + assert len(response) == 1 + + +@mock_ec2 +@mock_s3 +def test_s3_inactive_skip(): + """ + This method tests skip delete of the ami related snapshots + @return: + """ + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['policy'] = 's3_inactive' + tags = [{'Key': 'User', 'Value': TEST_USER_NAME}, {'Key': 'DaysCount', 'Value': f'{CURRENT_DATE}@7'}, + {'Key': 'policy', 'Value': 'not-delete'}] + s3_client = boto3.client('s3', region_name=AWS_DEFAULT_REGION) + s3_client.create_bucket(Bucket=TEST_USER_NAME, CreateBucketConfiguration={'LocationConstraint': AWS_DEFAULT_REGION}) + s3_client.put_bucket_tagging(Bucket=TEST_USER_NAME, Tagging={'TagSet': tags}) + s3_operations = S3Operations(region_name=AWS_DEFAULT_REGION) + # run s3_inactive + s3_inactive = S3Inactive() + response = s3_inactive.run() + assert len(s3_operations.list_buckets()) == 1 + assert len(response) == 1 + assert get_tag_value_from_tags(tags=s3_operations.get_bucket_tagging(bucket_name=TEST_USER_NAME), + tag_name='DaysCount') == f"{CURRENT_DATE}@7" + + +@mock_ec2 +@mock_s3 +def test_s3_inactive_contains_cluster_tag(): + """ + This method tests snapshot having the live cluster + @return: + """ + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['policy'] = 's3_inactive' + tags = [{'Key': 'User', 'Value': TEST_USER_NAME}, {'Key': 'DaysCount', 'Value': f'{CURRENT_DATE}@7'}, + {'Key': 'policy', 'Value': 'not-delete'}, + {'Key': 'kubernetes.io/cluster/test-zombie-cluster', 'Value': f'owned'}] + s3_client = boto3.client('s3', region_name=AWS_DEFAULT_REGION) + ec2_client = boto3.client('ec2', region_name=AWS_DEFAULT_REGION) + ec2_client.run_instances(ImageId=DEFAULT_AMI_ID, InstanceType=INSTANCE_TYPE_T2_MICRO, + MaxCount=1, MinCount=1, TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]) + s3_client.create_bucket(Bucket=TEST_USER_NAME, CreateBucketConfiguration={'LocationConstraint': AWS_DEFAULT_REGION}) + s3_client.put_bucket_tagging(Bucket=TEST_USER_NAME, Tagging={'TagSet': tags}) + s3_operations = S3Operations(region_name=AWS_DEFAULT_REGION) + # run s3_inactive + s3_inactive = S3Inactive() + response = s3_inactive.run() + assert len(s3_operations.list_buckets()) == 1 + assert len(response) == 0 + assert get_tag_value_from_tags(tags=s3_operations.get_bucket_tagging(bucket_name=TEST_USER_NAME), + tag_name='DaysCount') == f"{CURRENT_DATE}@0" + + +@mock_ec2 +@mock_s3 +def test_s3_inactive_contains_data(): + """ + This method tests snapshot having the live cluster + @return: + """ + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['policy'] = 's3_inactive' + tags = [{'Key': 'User', 'Value': TEST_USER_NAME}, {'Key': 'DaysCount', 'Value': f'{CURRENT_DATE}@7'}] + s3_client = boto3.client('s3', region_name=AWS_DEFAULT_REGION) + ec2_client = boto3.client('ec2', region_name=AWS_DEFAULT_REGION) + ec2_client.run_instances(ImageId=DEFAULT_AMI_ID, InstanceType=INSTANCE_TYPE_T2_MICRO, + MaxCount=1, MinCount=1, TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]) + s3_client.create_bucket(Bucket=TEST_USER_NAME, CreateBucketConfiguration={'LocationConstraint': AWS_DEFAULT_REGION}) + s3_client.put_bucket_tagging(Bucket=TEST_USER_NAME, Tagging={'TagSet': tags}) + s3_operations = S3Operations(region_name=AWS_DEFAULT_REGION) + with tempfile.NamedTemporaryFile(suffix='.txt') as file: + file_name = file.name.split('/')[-1] + s3_operations.upload_file(file_name_path=file.name, key='test', bucket=TEST_USER_NAME, upload_file=file_name) + # run s3_inactive + s3_inactive = S3Inactive() + response = s3_inactive.run() + assert len(s3_operations.list_buckets()) == 1 + assert len(response) == 0 + assert get_tag_value_from_tags(tags=s3_operations.get_bucket_tagging(bucket_name=TEST_USER_NAME), + tag_name='DaysCount') == f"{CURRENT_DATE}@0"