Added the database_idle policy (#780)

cloud_governance/common/clouds/aws/cloudwatch/cloudwatch_operations.py

@@ -1,6 +1,6 @@
-from datetime import datetime, timedelta
+from datetime import datetime
 
-import boto3
+from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client
 
 
 class CloudWatchOperations:
@@ -12,7 +12,7 @@ class CloudWatchOperations:
 
     def __init__(self, region: str = 'us-east-2'):
         self._region = region
-        self.cloudwatch_client = boto3.client('cloudwatch', region_name=self._region)
+        self.cloudwatch_client = get_boto3_client('cloudwatch', region_name=self._region)
 
     def _create_metric_lists(self, resource_id: str, resource_type: str, namespace: str, metric_names: dict, statistic: str):
         """

cloud_governance/common/clouds/aws/ec2/ec2_operations.py

@@ -4,6 +4,7 @@
 import typeguard
 from typing import Callable
 
+from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client
 from cloud_governance.common.clouds.aws.utils.utils import Utils
 from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
 from cloud_governance.main.environment_variables import environment_variables
@@ -21,9 +22,9 @@ def __init__(self, region: str = 'us-east-2'):
         Initializing the AWS resources
         """
         self.__environment_variables_dict = environment_variables.environment_variables_dict
-        self.elb1_client = boto3.client('elb', region_name=region)
-        self.elbv2_client = boto3.client('elbv2', region_name=region)
-        self.ec2_client = boto3.client('ec2', region_name=region)
+        self.elb1_client = get_boto3_client('elb', region_name=region)
+        self.elbv2_client = get_boto3_client('elbv2', region_name=region)
+        self.ec2_client = get_boto3_client('ec2', region_name=region)
         self.get_full_list = Utils().get_details_resource_list
         self.utils = Utils(region=region)
 

cloud_governance/common/clouds/aws/price/price.py

@@ -1,11 +1,10 @@
-
 from datetime import datetime
 from time import strftime
 
-import boto3
 import json
 from pkg_resources import resource_filename
 
+from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client
 from cloud_governance.common.logger.init_logger import logger
 # Search product filter
 from cloud_governance.main.environment_variables import environment_variables
@@ -29,7 +28,7 @@ class AWSPrice:
     def __init__(self, region_name: str = ''):
         # Use AWS Pricing API at US-East-1
         self.__environment_variables_dict = environment_variables.environment_variables_dict
-        self.__client = boto3.client('pricing', region_name='us-east-1')
+        self.__client = get_boto3_client('pricing', region_name='us-east-1')
         self.region = region_name if region_name else self.__environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-1')
 
     # Get current AWS price for an on-demand instance

cloud_governance/common/clouds/aws/price/resources_pricing.py

@@ -1,4 +1,3 @@
-
 from cloud_governance.common.clouds.aws.price.price import AWSPrice
 from cloud_governance.common.utils.configs import DEFAULT_ROUND_DIGITS
 from cloud_governance.main.environment_variables import environment_variables
@@ -90,3 +89,13 @@ def get_ebs_unit_price(self, region_name: str, ebs_type: str):
         ]
         unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict)
         return round(unit_price, DEFAULT_ROUND_DIGITS)
+
+    def get_rds_price(self, region_name: str, instance_type: str):
+        service_code = 'AmazonRDS'
+        filter_dict = [
+            {"Field": "productFamily", "Value": "Database Instance", "Type": "TERM_MATCH"},
+            {"Field": "regionCode", "Value": region_name, "Type": "TERM_MATCH"},
+            {"Field": "instanceType", "Value": instance_type, "Type": "TERM_MATCH"}
+        ]
+        unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict)
+        return round(unit_price, DEFAULT_ROUND_DIGITS)

cloud_governance/common/clouds/aws/rds/rds_operations.py

@@ -0,0 +1,43 @@
+from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client
+from cloud_governance.common.clouds.aws.utils.utils import Utils
+from cloud_governance.common.logger.init_logger import logger
+
+
+class RDSOperations:
+    """
+    This class performs the RDS operations
+    """
+
+    def __init__(self, region_name: str):
+        self._db_client = get_boto3_client('rds', region_name=region_name)
+
+    def describe_db_instances(self, **kwargs):
+        """
+        This method returns the rds databases
+        :return:
+        :rtype:
+        """
+        rds_instances = []
+        try:
+            rds_instances = Utils.iter_client_function(func_name=self._db_client.describe_db_instances,
+                                                       output_tag='DBInstances',
+                                                       iter_tag_name='Marker', **kwargs)
+        except Exception as err:
+            logger.error(f"Can't describe the rds instances: {err}")
+        return rds_instances
+
+    def add_tags_to_resource(self, resource_arn: str, tags: list):
+        """
+        This method add/ update the tags to the database
+        :param resource_arn:
+        :type resource_arn:
+        :param tags:
+        :type tags:
+        :return:
+        :rtype:
+        """
+        try:
+            self._db_client.add_tags_to_resource(ResourceName=resource_arn, Tags=tags)
+            logger.info(f"Tags are updated to the resource: {resource_arn}")
+        except Exception as err:
+            logger.error(f"Something went wrong in add/ update tags: {err}")

cloud_governance/common/clouds/aws/s3/s3_operations.py

@@ -3,12 +3,12 @@
 import json
 import os
 import tempfile
-import boto3
 import typeguard
 from botocore.exceptions import ClientError
 from os import listdir
 from os.path import isfile, join
 
+from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client
 from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
 
 
@@ -19,9 +19,9 @@ def __init__(self, region_name, report_file_name: str = "zombie_report.json",
                  resource_file_name: str = "resources.json.gz", bucket: str = '', logs_bucket_key: str = ''):
         #  @Todo ask AWS support regarding about this issue
         if region_name == 'eu-south-1':
-            self.__s3_client = boto3.client('s3', region_name='us-east-1')
+            self.__s3_client = get_boto3_client('s3', region_name='us-east-1')
         else:
-            self.__s3_client = boto3.client('s3', region_name=region_name)
+            self.__s3_client = get_boto3_client('s3', region_name=region_name)
         self.__region = region_name
         self.__report_file_name = report_file_name
         self.__resource_file_name = resource_file_name

cloud_governance/common/clouds/aws/utils/common_methods.py

@@ -1,3 +1,8 @@
+import boto3
+
+from cloud_governance.common.logger.init_logger import logger
+
+
 def get_tag_value_from_tags(tags: list, tag_name: str, cast_type: str = 'str',
                             default_value: any = '') -> any:
     """
@@ -26,3 +31,21 @@ def get_tag_value_from_tags(tags: list, tag_name: str, cast_type: str = 'str',
                         return str(tag.get('Value').strip())
                 return tag.get('Value').strip()
     return default_value
+
+
+def get_boto3_client(client: str, region_name: str, **kwargs):
+    """
+    This method initializes the aws boto3 client
+    :param client:
+    :type client:
+    :param region_name:
+    :type region_name:
+    :return:
+    :rtype:
+    """
+    client_object = None
+    try:
+        client_object = boto3.client(client, region_name=region_name, **kwargs)
+    except Exception as err:
+        logger.error(f"{client} Client Initialization error: {err}")
+    return client_object

cloud_governance/common/clouds/aws/utils/utils.py

@@ -81,3 +81,23 @@ def tag_aws_resources(self, client_method: Callable, tags: list, resource_ids: l
                     co += 1
             return co
 
+    @staticmethod
+    @typeguard.typechecked
+    def iter_client_function(func_name: Callable, output_tag: str, iter_tag_name: str, **kwargs):
+        """
+        This method fetch all Items of the resource i.e: EC2, IAM
+        :param func_name:
+        :param output_tag:
+        :param iter_tag_name:
+        :return:
+        """
+        resource_list = []
+        resources = func_name(**kwargs)
+        resource_list.extend(resources[output_tag])
+        while iter_tag_name in resources.keys():
+            if iter_tag_name == 'NextToken':
+                resources = func_name(NextToken=resources[iter_tag_name], **kwargs)
+            elif iter_tag_name == 'Marker':
+                resources = func_name(Marker=resources[iter_tag_name], **kwargs)
+            resource_list.extend(resources[output_tag])
+        return resource_list

cloud_governance/common/utils/configs.py

@@ -1,5 +1,3 @@
-
-
 LOOK_BACK_DAYS = 30
 MONTHS = 12
 DEFAULT_ROUND_DIGITS = 3
@@ -9,7 +7,6 @@
 HOURS_IN_MONTH = 720
 TOTAL_BYTES_IN_KIB = 1024
 
-
 DATE_FORMAT = "%Y-%m-%d"
 DATE_TIME_FORMAT_T = "%Y-%m-%dT%h:%m"
 UNUSED_DAYS = 7
@@ -21,3 +18,4 @@
 INSTANCE_IDLE_NETWORK_IN_KILO_BYTES = 5  # In KiB
 INSTANCE_IDLE_NETWORK_OUT_KILO_BYTES = 5  # In KiB
 EC2_NAMESPACE = 'AWS/EC2'
+CLOUDWATCH_METRICS_AVAILABLE_DAYS = 14

cloud_governance/main/environment_variables.py

@@ -1,7 +1,6 @@
 import argparse
 import os
 
-import tempfile
 from ast import literal_eval
 
 import boto3

cloud_governance/main/main_oerations/main_operations.py

@@ -1,4 +1,3 @@
-
 from cloud_governance.common.utils.utils import Utils
 from cloud_governance.main.environment_variables import environment_variables
 from cloud_governance.policy.policy_runners.azure.policy_runner import PolicyRunner as AzurePolicyRunner
@@ -40,7 +39,7 @@ def run(self):
             # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
             if self._policy in policies and self._policy in ["instance_run", "unattached_volume", "cluster_run",
                                                              "ip_unattached", "unused_nat_gateway", "instance_idle",
-                                                             "zombie_snapshots"]:
+                                                             "zombie_snapshots", "database_idle"]:
                 source = policy_type
                 if Utils.equal_ignore_case(policy_type, self._public_cloud_name):
                     source = ''

cloud_governance/policy/aws/cleanup/database_idle.py

@@ -0,0 +1,60 @@
+from cloud_governance.common.utils.configs import CLOUDWATCH_METRICS_AVAILABLE_DAYS
+from cloud_governance.common.utils.utils import Utils
+from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations
+
+
+class DatabaseIdle(AWSPolicyOperations):
+    """
+    This class performs the idle database operations
+    """
+
+    RESOURCE_ACTION = 'Delete'
+
+    def __init__(self):
+        super().__init__()
+
+    def run_policy_operations(self):
+        """
+        This method returns the idle databases
+        :return:
+        :rtype:
+        """
+        idle_dbs = []
+        dbs = self._rds_operations.describe_db_instances()
+        for db in dbs:
+            resource_id = db.get('DBInstanceIdentifier')
+            create_date = db.get('InstanceCreateTime')
+            tags = db.get('TagList', [])
+            cluster_tag = self._get_cluster_tag(tags=tags)
+            cleanup_result = False
+            running_days = self.calculate_days(create_date=create_date)
+            cleanup_days = 0
+            resource_arn = db.get('DBInstanceArn', '')
+            if Utils.greater_than(val1=running_days, val2=CLOUDWATCH_METRICS_AVAILABLE_DAYS) \
+                    and not cluster_tag \
+                    and self.is_database_idle(resource_id):
+                cleanup_days = self.get_clean_up_days_count(tags=tags)
+                cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags,
+                                                                 clean_up_days=cleanup_days)
+                unit_price = self._resource_pricing.get_rds_price(region_name=self._region,
+                                                                  instance_type=db.get('DBInstanceClass'))
+                resource_data = self._get_es_schema(resource_id=resource_id,
+                                                    user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                    skip_policy=self.get_skip_policy_value(tags=tags),
+                                                    cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                    name=db.get('DBName'),
+                                                    region=self._region,
+                                                    cleanup_result=str(cleanup_result),
+                                                    resource_action=self.RESOURCE_ACTION,
+                                                    cloud_name=self._cloud_name,
+                                                    launch_time=str(create_date),
+                                                    resource_type=db.get('DBInstanceClass'),
+                                                    unit_price=unit_price,
+                                                    resource_state=db.get('DBInstanceStatus')
+                                                    if not cleanup_result else "Deleted"
+                                                    )
+                idle_dbs.append(resource_data)
+            if not cleanup_result:
+                self.update_resource_day_count_tag(resource_id=resource_arn, cleanup_days=cleanup_days, tags=tags)
+
+        return idle_dbs

cloud_governance/policy/helpers/aws/aws_policy_operations.py

@@ -1,12 +1,12 @@
-
-import boto3
-
 from cloud_governance.common.clouds.aws.cloudwatch.cloudwatch_operations import CloudWatchOperations
 from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
 from cloud_governance.common.clouds.aws.price.resources_pricing import ResourcesPricing
+from cloud_governance.common.clouds.aws.rds.rds_operations import RDSOperations
 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
+from cloud_governance.common.clouds.aws.utils.common_methods import get_boto3_client
+from cloud_governance.common.clouds.aws.utils.utils import Utils
 from cloud_governance.common.utils.configs import INSTANCE_IDLE_DAYS, DEFAULT_ROUND_DIGITS, TOTAL_BYTES_IN_KIB, \
-    EC2_NAMESPACE
+    EC2_NAMESPACE, CLOUDWATCH_METRICS_AVAILABLE_DAYS
 from cloud_governance.common.utils.utils import Utils
 from cloud_governance.policy.helpers.abstract_policy_operations import AbstractPolicyOperations
 from cloud_governance.common.logger.init_logger import logger
@@ -18,13 +18,14 @@ def __init__(self):
         super().__init__()
         self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
         self._cloud_name = 'AWS'
+        self._ec2_client = get_boto3_client(client='ec2', region_name=self._region)
+        self._s3_client = get_boto3_client('s3', region_name=self._region)
+        self._iam_client = get_boto3_client('iam', region_name=self._region)
+        self._rds_operations = RDSOperations(region_name=self._region)
         self.__s3operations = S3Operations(region_name=self._region)
-        self._ec2_client = boto3.client('ec2', region_name=self._region)
         self._ec2_operations = EC2Operations(region=self._region)
         self._cloudwatch = CloudWatchOperations(region=self._region)
         self._resource_pricing = ResourcesPricing()
-        self._s3_client = boto3.client('s3')
-        self._iam_client = boto3.client('iam')
 
     def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
         """
@@ -67,6 +68,9 @@ def _delete_resource(self, resource_id: str):
             elif self._policy == 'instance_run':
                 self._ec2_client.stop_instances(InstanceIds=[resource_id])
                 action = "Stopped"
+            elif self._policy == 'database_idle':
+                # @ Todo add the delete method after successful monitoring
+                return False
             logger.info(f'{self._policy} {action}: {resource_id}')
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
@@ -111,7 +115,8 @@ def _update_tag_value(self, tags: list, tag_name: str, tag_value: str):
         tags = self.__remove_tag_key_aws(tags=tags)
         return tags
 
-    def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list, force_tag_update: str = ''):
+    def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list,
+                                      force_tag_update: str = ''):
         """
         This method updates the resource tags
         :param force_tag_update:
@@ -134,6 +139,8 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
             elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'unattached_volume',
                                   'instance_run', 'instance_idle'):
                 self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
+            elif self._policy == 'database_idle':
+                self._rds_operations.add_tags_to_resource(resource_arn=resource_id, tags=tags)
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
 
@@ -274,3 +281,25 @@ def _get_ami_ids(self):
         for image in images:
             image_ids.append(image.get('ImageId'))
         return image_ids
+
+    def __get_db_connection_status(self, resource_id: str, days: int = CLOUDWATCH_METRICS_AVAILABLE_DAYS):
+        start_date, end_date = Utils.get_start_and_end_datetime(days=days)
+        metrics = self._cloudwatch.get_metric_data(resource_id=resource_id, start_time=start_date, end_time=end_date,
+                                                   resource_type='DBInstanceIdentifier',
+                                                   metric_names={'DatabaseConnections': 'Count'},
+                                                   namespace='AWS/RDS', statistic='Maximum'
+                                                   )
+        total_connections = self.__get_aggregation_metrics_value(metrics.get('MetricDataResults', []),
+                                                                 aggregation='sum')
+        return total_connections
+
+    def is_database_idle(self, resource_id: str):
+        """
+        This method returns bool on verifying the database connections
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        total_connections = self.__get_db_connection_status(resource_id)
+        return int(total_connections) == 0