Added the azure unused_nat_gateway policy

README.md

@@ -34,7 +34,7 @@ This tool support the following policies:
 * [s3_inactive](cloud_governance/policy/aws/s3_inactive.py): Get the inactive/empty buckets and delete them after 7 days.
 * [empty_roles](cloud_governance/policy/aws/empty_roles.py): Get empty roles and delete it after 7 days.
 * [zombie_snapshots](cloud_governance/policy/aws/zombie_snapshots.py): Get the zombie snapshots and delete it after 7 days.
-* [unused_nat_gateway](cloud_governance/policy/aws/unused_nat_gateway.py): Get the unused nat gateways and deletes it after 7 days.
+* [unused_nat_gateway](cloud_governance/policy/aws/cleanup/unused_nat_gateway.py): Get the unused nat gateways and deletes it after 7 days.
 * gitleaks: scan Github repository git leak (security scan)  
 * [cost_over_usage](cloud_governance/policy/aws/cost_over_usage.py): send mail to aws user if over usage cost
 

cloud_governance/common/clouds/azure/compute/network_operations.py

@@ -1,6 +1,8 @@
+from azure.core.exceptions import HttpResponseError
 from azure.mgmt.network import NetworkManagementClient
 
 from cloud_governance.common.clouds.azure.common.common_operations import CommonOperations
+from cloud_governance.common.logger.init_logger import logger
 from cloud_governance.common.utils.utils import Utils
 
 
@@ -57,7 +59,24 @@ def get_public_ipv4_network_interfaces(self):
                     public_ipv4_network_interfaces.setdefault(public_ipv4_address_id, []).append(network_interface)
         return public_ipv4_network_interfaces
 
+    def describe_nat_gateways(self):
+        """
+        This method lists all the azure nat gateways
+        :return:
+        :rtype:
+        """
+        try:
+            nat_gateway_iter_object = self.__network_client.nat_gateways.list_all()
+            nat_gateways = self._item_paged_iterator(item_paged_object=nat_gateway_iter_object, as_dict=True)
+            return nat_gateways
+        except HttpResponseError as http_err:
+            logger.error(http_err)
+            raise http_err
+        except Exception as err:
+            raise err
+
     # delete operations
+
     def release_public_ip(self, resource_id: str):
         """
         This method releases the public ip
@@ -70,3 +89,18 @@ def release_public_ip(self, resource_id: str):
         status = self.__network_client.public_ip_addresses.begin_delete(resource_group_name=resource_group_name,
                                                                         public_ip_address_name=public_ip_address_name)
         return status.done()
+
+    def delete_nat_gateway(self, resource_id: str):
+        """
+        This method deletes the NatGateway
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        id_key_pairs = self.get_id_dict_data(resource_id)
+        resource_group_name = id_key_pairs.get('resourcegroups')
+        nat_gateway_name = id_key_pairs.get('natgateways')
+        status = self.__network_client.nat_gateways.begin_delete(resource_group_name=resource_group_name,
+                                                                 nat_gateway_name=nat_gateway_name)
+        return status.done()

cloud_governance/common/clouds/azure/monitor/monitor_management_operations.py

@@ -58,3 +58,32 @@ def get_audit_records(self, resource_id: str, start_date: datetime = None, end_d
             logger.error(http_error)
         except Exception as err:
             logger.error(err)
+
+    def get_resource_metrics(self, resource_id: str, metricnames: str, aggregation: str,
+                             timespan: str = None, interval: timedelta = timedelta(days=1), **kwargs):
+        """
+        This method returns the metrics object of individual resource
+        :param resource_id:
+        :type resource_id:
+        :param metricnames:
+        :type metricnames:
+        :param aggregation:
+        :type aggregation:
+        :param timespan:
+        :type timespan:
+        :param interval:
+        :type interval:
+        :param kwargs:
+        :type kwargs:
+        :return:
+        :rtype:
+        """
+        if not timespan:
+            end_date = datetime.utcnow()
+            start_date = end_date - timedelta(days=7)
+            timespan = f'{start_date}/{end_date}'
+        response = self.__monitor_client.metrics.list(resource_uri=resource_id, timespan=timespan,
+                                                      metricnames=metricnames, aggregation=aggregation,
+                                                      result_type='Data', interval=interval,
+                                                      **kwargs)
+        return response.as_dict()

cloud_governance/main/main_oerations/main_operations.py

@@ -39,7 +39,7 @@ def run(self):
         for policy_type, policies in policies_list.items():
             # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
             if self._policy in policies and self._policy in ["instance_run", "unattached_volume", "cluster_run",
-                                                             "ip_unattached"]:
+                                                             "ip_unattached", "unused_nat_gateway"]:
                 source = policy_type
                 if Utils.equal_ignore_case(policy_type, self._public_cloud_name):
                     source = ''

cloud_governance/policy/aws/cleanup/unused_nat_gateway.py

@@ -0,0 +1,92 @@
+import datetime
+
+from cloud_governance.common.utils.utils import Utils
+from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations
+
+
+class UnUsedNatGateway(AWSPolicyOperations):
+    """
+    This class sends an alert mail for zombie Nat gateways ( based on vpc routes )
+    to the user after 4 days and delete after 7 days.
+    """
+
+    NAMESPACE = 'AWS/NATGateway'
+    UNUSED_DAYS = 7
+    RESOURCE_ACTION = "Delete"
+
+    def __init__(self):
+        super().__init__()
+        self.__active_cluster_ids = self._get_active_cluster_ids()
+
+    def __check_cloud_watch_logs(self, resource_id: str, days: int = UNUSED_DAYS):
+        """
+        This method returns weather the NatGateway is used in last input days
+        :param resource_id:
+        :param days:
+        :return:
+        """
+        if days == 0:
+            days = 1
+        end_time = datetime.datetime.utcnow()
+        start_time = end_time - datetime.timedelta(days=days)
+        response = self._cloudwatch.get_metric_data(start_time=start_time, end_time=end_time, resource_id=resource_id,
+                                                    resource_type='NatGatewayId', namespace=self.NAMESPACE,
+                                                    metric_names={'ActiveConnectionCount': 'Count'},
+                                                    statistic='Average')['MetricDataResults'][0]
+        for value in response.get('Values', []):
+            if value > 0:
+                return False
+        return True
+
+    def __check_nat_gateway_in_routes(self, nat_gateway_id: str):
+        """
+        This method check the nat gateway present in the routes or not.
+        :param nat_gateway_id:
+        :return:
+        """
+        route_tables = self._ec2_client.describe_route_tables()['RouteTables']
+        nat_gateway_found = False
+        for route_table in route_tables:
+            for route in route_table.get('Routes'):
+                if route.get('NatGatewayId') == nat_gateway_id:
+                    nat_gateway_found = True
+        return nat_gateway_found
+
+    def run_policy_operations(self):
+        """
+        This method returns the list of unattached volumes
+        :return:
+        :rtype:
+        """
+        unused_nat_gateways = []
+        nat_gateways = self._ec2_operations.get_nat_gateways()
+        for nat_gateway in nat_gateways:
+            tags = nat_gateway.get('Tags', [])
+            resource_id = nat_gateway.get('NatGatewayId')
+            cleanup_result = False
+            cluster_tag = self._get_cluster_tag(tags=tags)
+            cleanup_days = 0
+            if (Utils.equal_ignore_case(nat_gateway.get('State'), 'available')
+                    and cluster_tag not in self.__active_cluster_ids):
+                if (not self.__check_nat_gateway_in_routes(nat_gateway_id=resource_id) or
+                        self.__check_cloud_watch_logs(resource_id=resource_id)):
+                    cleanup_days = self.get_clean_up_days_count(tags=tags)
+                    cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags,
+                                                                     clean_up_days=cleanup_days)
+                    resource_data = self._get_es_schema(resource_id=resource_id,
+                                                        user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                        skip_policy=self.get_skip_policy_value(tags=tags),
+                                                        cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                        name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
+                                                        region=self._region,
+                                                        cleanup_result=str(cleanup_result),
+                                                        resource_action=self.RESOURCE_ACTION,
+                                                        cloud_name=self._cloud_name,
+                                                        resource_type='NatGateway',
+                                                        resource_state=nat_gateway.get('State')
+                                                        if not cleanup_result else "Deleted")
+                    unused_nat_gateways.append(resource_data)
+            if not cleanup_result:
+                self.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags)
+
+        return unused_nat_gateways

cloud_governance/policy/azure/cleanup/unused_nat_gateway.py

@@ -0,0 +1,70 @@
+from cloud_governance.policy.helpers.azure.azure_policy_operations import AzurePolicyOperations
+
+
+class UnUsedNatGateway(AzurePolicyOperations):
+
+    """
+    This class performs the azure unused nat gateway operations
+    """
+
+    RESOURCE_ACTION = "Delete"
+
+    def __init__(self):
+        super().__init__()
+        self.__active_cluster_ids = self._get_active_cluster_ids()
+
+    def run_policy_operations(self):
+        """
+        This method returns the list of unused nat gateways
+        :return:
+        :rtype:
+        """
+        unused_nat_gateways = []
+        nat_gateways = self.network_operations.describe_nat_gateways()
+        for nat_gateway in nat_gateways:
+            tags = nat_gateway.get('tags')
+            cluster_tag = self._get_cluster_tag(tags=tags)
+            cleanup_result = False
+            if cluster_tag not in self.__active_cluster_ids:
+                metrics_data = self.monitor_operations.get_resource_metrics(resource_id=nat_gateway.get('id'),
+                                                                            metricnames='ByteCount',
+                                                                            aggregation='Average')
+                unused = False
+                if metrics_data.get('value'):
+                    metrics_time_series_data = metrics_data.get('value', [])[0].get('timeseries', [])
+                    if not metrics_time_series_data:
+                        unused = True
+                    else:
+                        total_data = 0
+                        for metric_time_frame in metrics_time_series_data:
+                            for data in metric_time_frame.get('data'):
+                                total_data = (total_data + data.get('average')) / 2
+                        if total_data < 5:
+                            unused = True
+
+                if unused:
+                    cleanup_days = self.get_clean_up_days_count(tags=tags)
+                    cleanup_result = self.verify_and_delete_resource(resource_id=nat_gateway.get('id'), tags=tags,
+                                                                     clean_up_days=cleanup_days)
+                    resource_data = self._get_es_schema(resource_id=nat_gateway.get('name'),
+                                                        user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
+                                                        skip_policy=self.get_skip_policy_value(tags=tags),
+                                                        cleanup_days=cleanup_days, dry_run=self._dry_run,
+                                                        name=nat_gateway.get('name'), region=nat_gateway.get('location'),
+                                                        cleanup_result=str(cleanup_result),
+                                                        resource_action=self.RESOURCE_ACTION,
+                                                        cloud_name=self._cloud_name,
+                                                        resource_type=f"NatGateway: "
+                                                                      f"{nat_gateway.get('sku', {}).get('name')}",
+                                                        resource_state=nat_gateway.get('provisioning_state')
+                                                        if not cleanup_result else "Deleted")
+                    unused_nat_gateways.append(resource_data)
+                else:
+                    cleanup_days = 0
+            else:
+                cleanup_days = 0
+            if not cleanup_result:
+                self.update_resource_day_count_tag(resource_id=nat_gateway.get("id"), cleanup_days=cleanup_days,
+                                                   tags=tags)
+
+        return unused_nat_gateways

cloud_governance/policy/helpers/aws/aws_policy_operations.py

@@ -1,6 +1,7 @@
 
 import boto3
 
+from cloud_governance.common.clouds.aws.cloudwatch.cloudwatch_operations import CloudWatchOperations
 from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
 from cloud_governance.policy.helpers.abstract_policy_operations import AbstractPolicyOperations
@@ -16,6 +17,7 @@ def __init__(self):
         self.__s3operations = S3Operations(region_name=self._region)
         self._ec2_client = boto3.client('ec2', region_name=self._region)
         self._ec2_operations = EC2Operations(region=self._region)
+        self._cloudwatch = CloudWatchOperations(region=self._region)
         self._s3_client = boto3.client('s3')
         self._iam_client = boto3.client('iam')