diff --git a/cloud_governance/common/clouds/aws/cloudwatch/cloudwatch_operations.py b/cloud_governance/common/clouds/aws/cloudwatch/cloudwatch_operations.py index 36c54b2a..373cbafb 100644 --- a/cloud_governance/common/clouds/aws/cloudwatch/cloudwatch_operations.py +++ b/cloud_governance/common/clouds/aws/cloudwatch/cloudwatch_operations.py @@ -1,6 +1,6 @@ -from datetime import datetime, timedelta +from datetime import datetime -import boto3 +from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client class CloudWatchOperations: @@ -12,7 +12,7 @@ class CloudWatchOperations: def __init__(self, region: str = 'us-east-2'): self._region = region - self.cloudwatch_client = boto3.client('cloudwatch', region_name=self._region) + self.cloudwatch_client = get_boto3_client('cloudwatch', region_name=self._region) def _create_metric_lists(self, resource_id: str, resource_type: str, namespace: str, metric_names: dict, statistic: str): """ diff --git a/cloud_governance/common/clouds/aws/ec2/ec2_operations.py b/cloud_governance/common/clouds/aws/ec2/ec2_operations.py index 1934cf1b..56b0ae2b 100644 --- a/cloud_governance/common/clouds/aws/ec2/ec2_operations.py +++ b/cloud_governance/common/clouds/aws/ec2/ec2_operations.py @@ -4,6 +4,7 @@ import typeguard from typing import Callable +from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client from cloud_governance.common.clouds.aws.utils.utils import Utils from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp from cloud_governance.main.environment_variables import environment_variables @@ -21,9 +22,9 @@ def __init__(self, region: str = 'us-east-2'): Initializing the AWS resources """ self.__environment_variables_dict = environment_variables.environment_variables_dict - self.elb1_client = boto3.client('elb', region_name=region) - self.elbv2_client = boto3.client('elbv2', region_name=region) - self.ec2_client = boto3.client('ec2', region_name=region) + self.elb1_client = get_boto3_client('elb', region_name=region) + self.elbv2_client = get_boto3_client('elbv2', region_name=region) + self.ec2_client = get_boto3_client('ec2', region_name=region) self.get_full_list = Utils().get_details_resource_list self.utils = Utils(region=region) diff --git a/cloud_governance/common/clouds/aws/price/price.py b/cloud_governance/common/clouds/aws/price/price.py index 0510015a..da03f902 100644 --- a/cloud_governance/common/clouds/aws/price/price.py +++ b/cloud_governance/common/clouds/aws/price/price.py @@ -1,11 +1,10 @@ - from datetime import datetime from time import strftime -import boto3 import json from pkg_resources import resource_filename +from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client from cloud_governance.common.logger.init_logger import logger # Search product filter from cloud_governance.main.environment_variables import environment_variables @@ -29,7 +28,7 @@ class AWSPrice: def __init__(self, region_name: str = ''): # Use AWS Pricing API at US-East-1 self.__environment_variables_dict = environment_variables.environment_variables_dict - self.__client = boto3.client('pricing', region_name='us-east-1') + self.__client = get_boto3_client('pricing', region_name='us-east-1') self.region = region_name if region_name else self.__environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-1') # Get current AWS price for an on-demand instance diff --git a/cloud_governance/common/clouds/aws/price/resources_pricing.py b/cloud_governance/common/clouds/aws/price/resources_pricing.py index 74518f6e..249fd940 100644 --- a/cloud_governance/common/clouds/aws/price/resources_pricing.py +++ b/cloud_governance/common/clouds/aws/price/resources_pricing.py @@ -1,4 +1,3 @@ - from cloud_governance.common.clouds.aws.price.price import AWSPrice from cloud_governance.common.utils.configs import DEFAULT_ROUND_DIGITS from cloud_governance.main.environment_variables import environment_variables @@ -90,3 +89,13 @@ def get_ebs_unit_price(self, region_name: str, ebs_type: str): ] unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict) return round(unit_price, DEFAULT_ROUND_DIGITS) + + def get_rds_price(self, region_name: str, instance_type: str): + service_code = 'AmazonRDS' + filter_dict = [ + {"Field": "productFamily", "Value": "Database Instance", "Type": "TERM_MATCH"}, + {"Field": "regionCode", "Value": region_name, "Type": "TERM_MATCH"}, + {"Field": "instanceType", "Value": instance_type, "Type": "TERM_MATCH"} + ] + unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict) + return round(unit_price, DEFAULT_ROUND_DIGITS) diff --git a/cloud_governance/common/clouds/aws/rds/__init__.py b/cloud_governance/common/clouds/aws/rds/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/clouds/aws/rds/rds_operations.py b/cloud_governance/common/clouds/aws/rds/rds_operations.py new file mode 100644 index 00000000..3001b52b --- /dev/null +++ b/cloud_governance/common/clouds/aws/rds/rds_operations.py @@ -0,0 +1,43 @@ +from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client +from cloud_governance.common.clouds.aws.utils.utils import Utils +from cloud_governance.common.logger.init_logger import logger + + +class RDSOperations: + """ + This class performs the RDS operations + """ + + def __init__(self, region_name: str): + self._db_client = get_boto3_client('rds', region_name=region_name) + + def describe_db_instances(self, **kwargs): + """ + This method returns the rds databases + :return: + :rtype: + """ + rds_instances = [] + try: + rds_instances = Utils.iter_client_function(func_name=self._db_client.describe_db_instances, + output_tag='DBInstances', + iter_tag_name='Marker', **kwargs) + except Exception as err: + logger.error(f"Can't describe the rds instances: {err}") + return rds_instances + + def add_tags_to_resource(self, resource_arn: str, tags: list): + """ + This method add/ update the tags to the database + :param resource_arn: + :type resource_arn: + :param tags: + :type tags: + :return: + :rtype: + """ + try: + self._db_client.add_tags_to_resource(ResourceName=resource_arn, Tags=tags) + logger.info(f"Tags are updated to the resource: {resource_arn}") + except Exception as err: + logger.error(f"Something went wrong in add/ update tags: {err}") diff --git a/cloud_governance/common/clouds/aws/s3/s3_operations.py b/cloud_governance/common/clouds/aws/s3/s3_operations.py index d3a5ef14..173cf4b8 100644 --- a/cloud_governance/common/clouds/aws/s3/s3_operations.py +++ b/cloud_governance/common/clouds/aws/s3/s3_operations.py @@ -3,12 +3,12 @@ import json import os import tempfile -import boto3 import typeguard from botocore.exceptions import ClientError from os import listdir from os.path import isfile, join +from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp @@ -19,9 +19,9 @@ def __init__(self, region_name, report_file_name: str = "zombie_report.json", resource_file_name: str = "resources.json.gz", bucket: str = '', logs_bucket_key: str = ''): # @Todo ask AWS support regarding about this issue if region_name == 'eu-south-1': - self.__s3_client = boto3.client('s3', region_name='us-east-1') + self.__s3_client = get_boto3_client('s3', region_name='us-east-1') else: - self.__s3_client = boto3.client('s3', region_name=region_name) + self.__s3_client = get_boto3_client('s3', region_name=region_name) self.__region = region_name self.__report_file_name = report_file_name self.__resource_file_name = resource_file_name diff --git a/cloud_governance/common/clouds/aws/utils/common_methods.py b/cloud_governance/common/clouds/aws/utils/common_methods.py index fbd1c6e6..a2ff8da8 100644 --- a/cloud_governance/common/clouds/aws/utils/common_methods.py +++ b/cloud_governance/common/clouds/aws/utils/common_methods.py @@ -1,3 +1,8 @@ +import boto3 + +from cloud_governance.common.logger.init_logger import logger + + def get_tag_value_from_tags(tags: list, tag_name: str, cast_type: str = 'str', default_value: any = '') -> any: """ @@ -26,3 +31,21 @@ def get_tag_value_from_tags(tags: list, tag_name: str, cast_type: str = 'str', return str(tag.get('Value').strip()) return tag.get('Value').strip() return default_value + + +def get_boto3_client(client: str, region_name: str, **kwargs): + """ + This method initializes the aws boto3 client + :param client: + :type client: + :param region_name: + :type region_name: + :return: + :rtype: + """ + client_object = None + try: + client_object = boto3.client(client, region_name=region_name, **kwargs) + except Exception as err: + logger.error(f"{client} Client Initialization error: {err}") + return client_object diff --git a/cloud_governance/common/clouds/aws/utils/utils.py b/cloud_governance/common/clouds/aws/utils/utils.py index 0c59ec7c..2103ad84 100644 --- a/cloud_governance/common/clouds/aws/utils/utils.py +++ b/cloud_governance/common/clouds/aws/utils/utils.py @@ -81,3 +81,23 @@ def tag_aws_resources(self, client_method: Callable, tags: list, resource_ids: l co += 1 return co + @staticmethod + @typeguard.typechecked + def iter_client_function(func_name: Callable, output_tag: str, iter_tag_name: str, **kwargs): + """ + This method fetch all Items of the resource i.e: EC2, IAM + :param func_name: + :param output_tag: + :param iter_tag_name: + :return: + """ + resource_list = [] + resources = func_name(**kwargs) + resource_list.extend(resources[output_tag]) + while iter_tag_name in resources.keys(): + if iter_tag_name == 'NextToken': + resources = func_name(NextToken=resources[iter_tag_name], **kwargs) + elif iter_tag_name == 'Marker': + resources = func_name(Marker=resources[iter_tag_name], **kwargs) + resource_list.extend(resources[output_tag]) + return resource_list diff --git a/cloud_governance/common/utils/configs.py b/cloud_governance/common/utils/configs.py index 1e09c1e7..742bb1cf 100644 --- a/cloud_governance/common/utils/configs.py +++ b/cloud_governance/common/utils/configs.py @@ -1,5 +1,3 @@ - - LOOK_BACK_DAYS = 30 MONTHS = 12 DEFAULT_ROUND_DIGITS = 3 @@ -9,7 +7,6 @@ HOURS_IN_MONTH = 720 TOTAL_BYTES_IN_KIB = 1024 - DATE_FORMAT = "%Y-%m-%d" DATE_TIME_FORMAT_T = "%Y-%m-%dT%h:%m" UNUSED_DAYS = 7 @@ -21,3 +18,4 @@ INSTANCE_IDLE_NETWORK_IN_KILO_BYTES = 5 # In KiB INSTANCE_IDLE_NETWORK_OUT_KILO_BYTES = 5 # In KiB EC2_NAMESPACE = 'AWS/EC2' +CLOUDWATCH_METRICS_AVAILABLE_DAYS = 14 diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index 1fa66d3c..630fdfc1 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -1,7 +1,6 @@ import argparse import os -import tempfile from ast import literal_eval import boto3 diff --git a/cloud_governance/main/main_oerations/main_operations.py b/cloud_governance/main/main_oerations/main_operations.py index 0f8c0a34..bd7c5094 100644 --- a/cloud_governance/main/main_oerations/main_operations.py +++ b/cloud_governance/main/main_oerations/main_operations.py @@ -1,4 +1,3 @@ - from cloud_governance.common.utils.utils import Utils from cloud_governance.main.environment_variables import environment_variables from cloud_governance.policy.policy_runners.azure.policy_runner import PolicyRunner as AzurePolicyRunner @@ -40,7 +39,7 @@ def run(self): # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement if self._policy in policies and self._policy in ["instance_run", "unattached_volume", "cluster_run", "ip_unattached", "unused_nat_gateway", "instance_idle", - "zombie_snapshots"]: + "zombie_snapshots", "database_idle"]: source = policy_type if Utils.equal_ignore_case(policy_type, self._public_cloud_name): source = '' diff --git a/cloud_governance/policy/aws/cleanup/database_idle.py b/cloud_governance/policy/aws/cleanup/database_idle.py new file mode 100644 index 00000000..0de14f24 --- /dev/null +++ b/cloud_governance/policy/aws/cleanup/database_idle.py @@ -0,0 +1,60 @@ +from cloud_governance.common.utils.configs import CLOUDWATCH_METRICS_AVAILABLE_DAYS +from cloud_governance.common.utils.utils import Utils +from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations + + +class DatabaseIdle(AWSPolicyOperations): + """ + This class performs the idle database operations + """ + + RESOURCE_ACTION = 'Delete' + + def __init__(self): + super().__init__() + + def run_policy_operations(self): + """ + This method returns the idle databases + :return: + :rtype: + """ + idle_dbs = [] + dbs = self._rds_operations.describe_db_instances() + for db in dbs: + resource_id = db.get('DBInstanceIdentifier') + create_date = db.get('InstanceCreateTime') + tags = db.get('TagList', []) + cluster_tag = self._get_cluster_tag(tags=tags) + cleanup_result = False + running_days = self.calculate_days(create_date=create_date) + cleanup_days = 0 + resource_arn = db.get('DBInstanceArn', '') + if Utils.greater_than(val1=running_days, val2=CLOUDWATCH_METRICS_AVAILABLE_DAYS) \ + and not cluster_tag \ + and self.is_database_idle(resource_id): + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags, + clean_up_days=cleanup_days) + unit_price = self._resource_pricing.get_rds_price(region_name=self._region, + instance_type=db.get('DBInstanceClass')) + resource_data = self._get_es_schema(resource_id=resource_id, + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + skip_policy=self.get_skip_policy_value(tags=tags), + cleanup_days=cleanup_days, dry_run=self._dry_run, + name=db.get('DBName'), + region=self._region, + cleanup_result=str(cleanup_result), + resource_action=self.RESOURCE_ACTION, + cloud_name=self._cloud_name, + launch_time=str(create_date), + resource_type=db.get('DBInstanceClass'), + unit_price=unit_price, + resource_state=db.get('DBInstanceStatus') + if not cleanup_result else "Deleted" + ) + idle_dbs.append(resource_data) + if not cleanup_result: + self.update_resource_day_count_tag(resource_id=resource_arn, cleanup_days=cleanup_days, tags=tags) + + return idle_dbs diff --git a/cloud_governance/policy/helpers/aws/aws_policy_operations.py b/cloud_governance/policy/helpers/aws/aws_policy_operations.py index 0526b911..69dba0a3 100644 --- a/cloud_governance/policy/helpers/aws/aws_policy_operations.py +++ b/cloud_governance/policy/helpers/aws/aws_policy_operations.py @@ -1,12 +1,12 @@ - -import boto3 - from cloud_governance.common.clouds.aws.cloudwatch.cloudwatch_operations import CloudWatchOperations from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations from cloud_governance.common.clouds.aws.price.resources_pricing import ResourcesPricing +from cloud_governance.common.clouds.aws.rds.rds_operations import RDSOperations from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations +from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client +from cloud_governance.common.clouds.aws.utils.utils import Utils from cloud_governance.common.utils.configs import INSTANCE_IDLE_DAYS, DEFAULT_ROUND_DIGITS, TOTAL_BYTES_IN_KIB, \ - EC2_NAMESPACE + EC2_NAMESPACE, CLOUDWATCH_METRICS_AVAILABLE_DAYS from cloud_governance.common.utils.utils import Utils from cloud_governance.policy.helpers.abstract_policy_operations import AbstractPolicyOperations from cloud_governance.common.logger.init_logger import logger @@ -18,13 +18,14 @@ def __init__(self): super().__init__() self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') self._cloud_name = 'AWS' + self._ec2_client = get_boto3_client(client='ec2', region_name=self._region) + self._s3_client = get_boto3_client('s3', region_name=self._region) + self._iam_client = get_boto3_client('iam', region_name=self._region) + self._rds_operations = RDSOperations(region_name=self._region) self.__s3operations = S3Operations(region_name=self._region) - self._ec2_client = boto3.client('ec2', region_name=self._region) self._ec2_operations = EC2Operations(region=self._region) self._cloudwatch = CloudWatchOperations(region=self._region) self._resource_pricing = ResourcesPricing() - self._s3_client = boto3.client('s3') - self._iam_client = boto3.client('iam') def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str: """ @@ -67,6 +68,9 @@ def _delete_resource(self, resource_id: str): elif self._policy == 'instance_run': self._ec2_client.stop_instances(InstanceIds=[resource_id]) action = "Stopped" + elif self._policy == 'database_idle': + # @ Todo add the delete method after successful monitoring + return False logger.info(f'{self._policy} {action}: {resource_id}') except Exception as err: logger.info(f'Exception raised: {err}: {resource_id}') @@ -111,7 +115,8 @@ def _update_tag_value(self, tags: list, tag_name: str, tag_value: str): tags = self.__remove_tag_key_aws(tags=tags) return tags - def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list, force_tag_update: str = ''): + def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list, + force_tag_update: str = ''): """ This method updates the resource tags :param force_tag_update: @@ -134,6 +139,8 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'unattached_volume', 'instance_run', 'instance_idle'): self._ec2_client.create_tags(Resources=[resource_id], Tags=tags) + elif self._policy == 'database_idle': + self._rds_operations.add_tags_to_resource(resource_arn=resource_id, tags=tags) except Exception as err: logger.info(f'Exception raised: {err}: {resource_id}') @@ -274,3 +281,25 @@ def _get_ami_ids(self): for image in images: image_ids.append(image.get('ImageId')) return image_ids + + def __get_db_connection_status(self, resource_id: str, days: int = CLOUDWATCH_METRICS_AVAILABLE_DAYS): + start_date, end_date = Utils.get_start_and_end_datetime(days=days) + metrics = self._cloudwatch.get_metric_data(resource_id=resource_id, start_time=start_date, end_time=end_date, + resource_type='DBInstanceIdentifier', + metric_names={'DatabaseConnections': 'Count'}, + namespace='AWS/RDS', statistic='Maximum' + ) + total_connections = self.__get_aggregation_metrics_value(metrics.get('MetricDataResults', []), + aggregation='sum') + return total_connections + + def is_database_idle(self, resource_id: str): + """ + This method returns bool on verifying the database connections + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + total_connections = self.__get_db_connection_status(resource_id) + return int(total_connections) == 0 diff --git a/tests/unittest/cloud_governance/aws/dynamodb_upload_data/test_upload_cloudtrail_data_to_dynamodb.py b/tests/unittest/cloud_governance/aws/dynamodb_upload_data/test_upload_cloudtrail_data_to_dynamodb.py index d075da89..8213adcc 100644 --- a/tests/unittest/cloud_governance/aws/dynamodb_upload_data/test_upload_cloudtrail_data_to_dynamodb.py +++ b/tests/unittest/cloud_governance/aws/dynamodb_upload_data/test_upload_cloudtrail_data_to_dynamodb.py @@ -5,7 +5,7 @@ import boto3 import pytest -from moto import mock_cloudtrail, mock_ec2, mock_dynamodb2, mock_iam +from moto import mock_cloudtrail, mock_ec2, mock_dynamodb, mock_iam from cloud_governance.policy.policy_operations.aws.dynamodb_upload_data.upload_data_to_dynamodb import UploadDataToDynamoDb from cloud_governance.policy.aws.ec2_stop import EC2Stop @@ -18,7 +18,7 @@ @mock_ec2 @mock_cloudtrail -@mock_dynamodb2 +@mock_dynamodb @pytest.mark.skip(reason="May be enabled in future") def test_upload_data_ec2_stop(): """ @@ -56,7 +56,7 @@ def test_upload_data_ec2_stop(): @pytest.mark.skip(reason="May be enabled in future") @mock_ec2 @mock_cloudtrail -@mock_dynamodb2 +@mock_dynamodb @mock_iam def test_upload_data_tag_non_cluster_resource(): iam_client = boto3.client('iam') @@ -85,7 +85,7 @@ def test_upload_data_tag_non_cluster_resource(): assert len(tag_resources.non_cluster_update_ec2()) == 1 -@mock_dynamodb2 +@mock_dynamodb def test_upload_data(): """ This method test data is uploaded to table correctly or not diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_database_idle.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_database_idle.py new file mode 100644 index 00000000..20f50676 --- /dev/null +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_database_idle.py @@ -0,0 +1,118 @@ +from datetime import datetime, timedelta + +from freezegun import freeze_time +from moto import mock_rds, mock_cloudwatch + +from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client, get_tag_value_from_tags +from cloud_governance.policy.aws.cleanup.database_idle import DatabaseIdle +from cloud_governance.main.environment_variables import environment_variables +from tests.unittest.configs import DB_INSTANCE_CLASS, AWS_DEFAULT_REGION, TEST_USER_NAME, DB_ENGINE, \ + CLOUD_WATCH_METRICS_DAYS, PROJECT_NAME + +current_date = datetime.utcnow() +start_date = current_date - timedelta(days=CLOUD_WATCH_METRICS_DAYS + 1) + + +@mock_cloudwatch +@mock_rds +@freeze_time(start_date.__str__()) +def test_database_idle(): + """ + This method tests database_idle resources + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + environment_variables.environment_variables_dict['policy'] = 'database_idle' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + rds_client = get_boto3_client('rds', region_name=AWS_DEFAULT_REGION) + tags = [{'Key': 'User', 'Value': PROJECT_NAME}, {'Key': "Name", "Value": TEST_USER_NAME}] + rds_client.create_db_instance(DBInstanceIdentifier=TEST_USER_NAME, + DBInstanceClass=DB_INSTANCE_CLASS, + Engine=DB_ENGINE, Tags=tags) + database_idle = DatabaseIdle() + running_instances_data = database_idle.run() + assert len(running_instances_data) == 1 + assert get_tag_value_from_tags(tags=rds_client.describe_db_instances()['DBInstances'][0]['TagList'], + tag_name='DaysCount') == f"{current_date.date()}@1" + + +@mock_cloudwatch +@mock_rds +@freeze_time(start_date.__str__()) +def test_database_idle_alert_skip(): + """ + This method tests database_idle skip delete + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 + environment_variables.environment_variables_dict['policy'] = 'database_idle' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + rds_client = get_boto3_client('rds', region_name=AWS_DEFAULT_REGION) + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': PROJECT_NAME}, {'Key': "Name", "Value": TEST_USER_NAME}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}, {'Key': "Skip", "Value": f"notdelete"}] + rds_client.create_db_instance(DBInstanceIdentifier=TEST_USER_NAME, + DBInstanceClass=DB_INSTANCE_CLASS, + Engine=DB_ENGINE, Tags=tags) + database_idle = DatabaseIdle() + running_instances_data = database_idle.run() + assert running_instances_data[0]['DryRun'] == 'no' + assert len(rds_client.describe_db_instances()['DBInstances']) == 1 + + +@mock_cloudwatch +@mock_rds +@freeze_time(start_date.__str__()) +def test_database_idle_delete(): + """ + This method tests the deletion of database_idle + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 + environment_variables.environment_variables_dict['policy'] = 'database_idle' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + rds_client = get_boto3_client('rds', region_name=AWS_DEFAULT_REGION) + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': PROJECT_NAME}, {'Key': "Name", "Value": TEST_USER_NAME}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}] + rds_client.create_db_instance(DBInstanceIdentifier=TEST_USER_NAME, + DBInstanceClass=DB_INSTANCE_CLASS, + Engine=DB_ENGINE, Tags=tags) + database_idle = DatabaseIdle() + running_instances_data = database_idle.run() + assert running_instances_data[0]['DryRun'] == 'no' + assert running_instances_data[0]['ResourceDelete'] == 'True' + + +@mock_cloudwatch +@mock_rds +@freeze_time(start_date.__str__()) +def test_database_idle_dry_run_yes(): + """ + This method tests the deletion of database_idle + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 + environment_variables.environment_variables_dict['policy'] = 'database_idle' + environment_variables.environment_variables_dict['dry_run'] = 'yes' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + rds_client = get_boto3_client('rds', region_name=AWS_DEFAULT_REGION) + mock_date = (datetime.now() - timedelta(days=2)).date() + tags = [{'Key': 'User', 'Value': PROJECT_NAME}, {'Key': "Name", "Value": TEST_USER_NAME}, + {'Key': "DaysCount", "Value": f"{mock_date}@3"}] + rds_client.create_db_instance(DBInstanceIdentifier=TEST_USER_NAME, + DBInstanceClass=DB_INSTANCE_CLASS, + Engine=DB_ENGINE, Tags=tags) + database_idle = DatabaseIdle() + running_instances_data = database_idle.run() + assert running_instances_data[0]['DryRun'] == 'yes' + assert running_instances_data[0]['ResourceDelete'] == 'False' + assert get_tag_value_from_tags(tags=rds_client.describe_db_instances()['DBInstances'][0]['TagList'], + tag_name='DaysCount') == f"{current_date.date()}@0" diff --git a/tests/unittest/configs.py b/tests/unittest/configs.py index e8b3643d..26a8eea6 100644 --- a/tests/unittest/configs.py +++ b/tests/unittest/configs.py @@ -1,5 +1,6 @@ import datetime +PROJECT_NAME = 'cloud_governance' # Common Variables @@ -9,7 +10,6 @@ CURRENT_DATE_TIME = datetime.datetime.utcnow() TEST_USER_NAME = 'unit-test' - # AWS AWS_DEFAULT_REGION = 'us-west-2' DEFAULT_AMI_ID = 'ami-03cf127a' @@ -17,7 +17,10 @@ INSTANCE_TYPE_T2_MICRO = 't2.micro' NAT_GATEWAY_NAMESPACE = 'AWS/NATGateway' EC2_NAMESPACE = 'AWS/EC2' - +CLOUD_WATCH_METRICS_DAYS = 14 +# AWS RDS +DB_INSTANCE_CLASS = 'db.t3.small' +DB_ENGINE = 'postgres' # Azure SUBSCRIPTION_ID = 'unitest-subscription' @@ -28,7 +31,6 @@ AZURE_RESOURCE_ID = f'{SUB_ID}/{COMPUTE_PROVIDER}/cloud-governance-unittest' NAT_GATEWAY_NAME = 'test-cloud-governance-nat-1' - # ES ES_INDEX = 'test-unittest-index' TEST_INDEX_ID = 'test-unittest-index-01' diff --git a/tests_requirements.txt b/tests_requirements.txt index acc5b04f..2056823f 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -7,7 +7,8 @@ azure-mgmt-subscription==3.1.1 boto3==1.26.4 elasticsearch==7.11.0 elasticsearch-dsl==7.4.0 -moto==2.3.2 +freezegun==1.5.1 +moto==4.0.1 oauthlib~=3.1.1 pandas pytest