From 1e0c5f8a80fcad771cf7ecb327ec1b41036f6d93 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Thu, 5 Sep 2024 14:00:28 +0530 Subject: [PATCH] Add the unit price to policies --- .../clouds/aws/price/resources_pricing.py | 46 +++++++++++++++++++ .../policy/aws/cleanup/instance_idle.py | 7 ++- .../policy/aws/cleanup/instance_run.py | 6 ++- .../policy/aws/zombie_snapshots.py | 7 ++- .../common/clouds/aws/price/__init__.py | 0 .../common/clouds/aws/price/test_price.py | 14 ++++++ .../policy/aws/cleanup/test_instance_idle.py | 31 +++++++------ 7 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 tests/integration/cloud_governance/common/clouds/aws/price/__init__.py create mode 100644 tests/integration/cloud_governance/common/clouds/aws/price/test_price.py diff --git a/cloud_governance/common/clouds/aws/price/resources_pricing.py b/cloud_governance/common/clouds/aws/price/resources_pricing.py index 249fd940..fe70fa4f 100644 --- a/cloud_governance/common/clouds/aws/price/resources_pricing.py +++ b/cloud_governance/common/clouds/aws/price/resources_pricing.py @@ -99,3 +99,49 @@ def get_rds_price(self, region_name: str, instance_type: str): ] unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict) return round(unit_price, DEFAULT_ROUND_DIGITS) + + def get_snapshot_unit_price(self, region_name: str): + """ + This method returns the unit price of Ebs Snapshot + :param region_name: + :return: + """ + service_code = 'AmazonEC2' + filter_dict = [ + {"Field": "regionCode", "Value": region_name, "Type": "TERM_MATCH"}, + {"Field": "productFamily", "Value": "Storage Snapshot", "Type": "TERM_MATCH"}, + {"Field": "snapshotarchivefeetype", "Value": "SnapshotArchiveStorage", "Type": "TERM_MATCH"}, + ] + unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict) + return round(unit_price, DEFAULT_ROUND_DIGITS) + + def get_ec2_price(self, region_name: str, instance_type: str, operating_system: str = None): + """ + This method returns the unit price of Ec2 Price + :param operating_system: + :param region_name: + :param instance_type: + :return: + """ + if not operating_system: + operating_system = 'Linux/UNIX' + os_types = {'Linux/UNIX': 'Linux', + 'Red Hat Enterprise Linux': 'RHEL', + 'SUSE Linux': 'SUSE', + 'Ubuntu Pro Linux': 'Ubuntu Pro', + 'Windows': 'Windows', + 'Red Hat Enterprise Linux with High Availability': 'Red Hat Enterprise Linux with HA'} + operating_system_value = "NA" + for os_type in os_types.keys(): + if os_type.lower() == operating_system.lower(): + operating_system_value = os_types[os_type] + service_code = 'AmazonEC2' + filter_dict = [ + {"Field": "regionCode", "Value": region_name, "Type": "TERM_MATCH"}, + {"Field": "tenancy", "Value": "shared", "Type": "TERM_MATCH"}, + {"Field": "operatingSystem", "Value": f"{operating_system_value}", "Type": "TERM_MATCH"}, + {"Field": "instanceType", "Value": f"{instance_type}", "Type": "TERM_MATCH"}, + {"Field": "capacitystatus", "Value": "Used", "Type": "TERM_MATCH"}, + ] + unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict) + return round(unit_price, DEFAULT_ROUND_DIGITS) diff --git a/cloud_governance/policy/aws/cleanup/instance_idle.py b/cloud_governance/policy/aws/cleanup/instance_idle.py index 54f63930..b7327462 100644 --- a/cloud_governance/policy/aws/cleanup/instance_idle.py +++ b/cloud_governance/policy/aws/cleanup/instance_idle.py @@ -1,4 +1,3 @@ - from cloud_governance.common.utils.configs import INSTANCE_IDLE_DAYS from cloud_governance.common.utils.utils import Utils from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations @@ -38,6 +37,9 @@ def run_policy_operations(self): self.get_skip_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP') and \ self.verify_instance_idle(resource_id=instance_id): cleanup_days = self.get_clean_up_days_count(tags=tags) + unit_price = self._resource_pricing.get_ec2_price(region_name=self._region, + instance_type=instance.get('InstanceType'), + operating_system=instance.get('PlatformDetails')) cleanup_result = self.verify_and_delete_resource(resource_id=instance_id, tags=tags, clean_up_days=cleanup_days) resource_data = self._get_es_schema( @@ -52,7 +54,8 @@ def run_policy_operations(self): name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'), resource_action=self.RESOURCE_ACTION, region=self._region, cleanup_result=str(cleanup_result), - cloud_name=self._cloud_name + cloud_name=self._cloud_name, + unit_price=unit_price ) if self._force_delete and self._dry_run == 'no': resource_data.update({'ForceDeleted': str(self._force_delete)}) diff --git a/cloud_governance/policy/aws/cleanup/instance_run.py b/cloud_governance/policy/aws/cleanup/instance_run.py index 4ff7af3e..5b09323b 100644 --- a/cloud_governance/policy/aws/cleanup/instance_run.py +++ b/cloud_governance/policy/aws/cleanup/instance_run.py @@ -4,7 +4,6 @@ class InstanceRun(AWSPolicyOperations): - INSTANCE_TYPES_ES_INDEX = "cloud-governance-instance-types" RESOURCE_ACTION = "Stopped" @@ -62,8 +61,12 @@ def run_policy_operations(self): for instance in instances: tags = instance.get('Tags', []) cleanup_result = False + if instance.get('State', {}).get('Name') == 'running': running_days = self.calculate_days(instance.get('LaunchTime')) + unit_price = self._resource_pricing.get_ec2_price(region_name=self._region, + instance_type=instance.get('InstanceType'), + operating_system=instance.get('PlatformDetails')) if self._shutdown_period: cleanup_days = self.get_clean_up_days_count(tags=tags) cleanup_result = self.verify_and_delete_resource( @@ -84,6 +87,7 @@ def run_policy_operations(self): name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'), region=self._region, cleanup_result=str(cleanup_result), cloud_name=self._cloud_name, + unit_price=unit_price ) if self._shutdown_period and self._force_delete and self._dry_run == 'no': resource_data.update({'ForceDeleted': str(self._force_delete)}) diff --git a/cloud_governance/policy/aws/zombie_snapshots.py b/cloud_governance/policy/aws/zombie_snapshots.py index 66668d7b..d4b6e9b5 100644 --- a/cloud_governance/policy/aws/zombie_snapshots.py +++ b/cloud_governance/policy/aws/zombie_snapshots.py @@ -1,3 +1,4 @@ +from cloud_governance.common.utils.configs import HOURS_IN_MONTH from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations @@ -32,6 +33,7 @@ def run(self): This method returns all the zombie snapshots and delete after x days @return: """ + monthly_price = self._resource_pricing.get_snapshot_unit_price(region_name=self._region) snapshots = self._ec2_operations.get_snapshots() zombie_snapshots = [] for snapshot in snapshots: @@ -45,7 +47,7 @@ def run(self): cleanup_days = self.get_clean_up_days_count(tags=tags) cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags, clean_up_days=cleanup_days) - unit_price = 0 + unit_price = (monthly_price / HOURS_IN_MONTH) * float(snapshot.get('VolumeSize')) resource_data = self._get_es_schema(resource_id=resource_id, user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), skip_policy=self.get_skip_policy_value(tags=tags), @@ -57,7 +59,8 @@ def run(self): cloud_name=self._cloud_name, resource_type='Snapshot', volume_size=f"{snapshot.get('VolumeSize')} GB", - unit_price=unit_price, resource_state='Backup' if not cleanup_result else "Deleted" + unit_price=unit_price, + resource_state='Backup' if not cleanup_result else "Deleted" ) zombie_snapshots.append(resource_data) if not cleanup_result: diff --git a/tests/integration/cloud_governance/common/clouds/aws/price/__init__.py b/tests/integration/cloud_governance/common/clouds/aws/price/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/cloud_governance/common/clouds/aws/price/test_price.py b/tests/integration/cloud_governance/common/clouds/aws/price/test_price.py new file mode 100644 index 00000000..b7a98f42 --- /dev/null +++ b/tests/integration/cloud_governance/common/clouds/aws/price/test_price.py @@ -0,0 +1,14 @@ +from cloud_governance.common.clouds.aws.price.price import AWSPrice + + +def test_get_service_pricing(): + aws_price = AWSPrice() + service_code = 'AmazonEC2' + filter_list = [ + {"Field": "regionCode", "Value": 'ap-south-1', "Type": "TERM_MATCH"}, + {"Field": "tenancy", "Value": "shared", "Type": "TERM_MATCH"}, + {"Field": "operatingSystem", "Value": "Linux", "Type": "TERM_MATCH"}, + {"Field": "instanceType", "Value": "t2.micro", "Type": "TERM_MATCH"}, + {"Field": "capacitystatus", "Value": "Used", "Type": "TERM_MATCH"}, + ] + assert aws_price.get_service_pricing(service_code, filter_list) > 0 diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_idle.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_idle.py index a51653ec..73057d76 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_idle.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_idle.py @@ -1,4 +1,3 @@ - from datetime import datetime, timedelta from typing import Union from unittest.mock import patch @@ -41,7 +40,8 @@ def mock_describe_instances(*args, **kwargs): 'InstanceId': 'i-1234567890abcdef0', 'State': {'Name': 'running'}, 'LaunchTime': kwargs.get('LaunchTime', datetime.utcnow()), - 'Tags': kwargs.get('Tags', []) + 'Tags': kwargs.get('Tags', []), + 'PlatformDetails': 'Linux/UNIX' # Change the launch time here } ] @@ -75,11 +75,13 @@ def test_instance_idle__check_not_idle(): environment_variables.environment_variables_dict['dry_run'] = 'yes' environment_variables.environment_variables_dict['policy'] = 'instance_idle' with patch('boto3.client') as mock_client: - mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(LaunchTime=datetime.utcnow() - timedelta(days=8))] - mock_client.return_value.get_metric_data.side_effect = [MockCloudWatchMetric(metrics=[5, 4, 8, 10]).create_metric(), - MockCloudWatchMetric(metrics=[5000, 2000, 4000, 8000]).create_metric(), - MockCloudWatchMetric(metrics=[1000, 200, 500]).create_metric() - ] + mock_client.return_value.describe_instances.side_effect = [ + mock_describe_instances(LaunchTime=datetime.utcnow() - timedelta(days=8))] + mock_client.return_value.get_metric_data.side_effect = [ + MockCloudWatchMetric(metrics=[5, 4, 8, 10]).create_metric(), + MockCloudWatchMetric(metrics=[5000, 2000, 4000, 8000]).create_metric(), + MockCloudWatchMetric(metrics=[1000, 200, 500]).create_metric() + ] instance_idle = InstanceIdle() response = instance_idle.run() assert len(response) == 0 @@ -95,7 +97,8 @@ def test_instance_idle__skip_cluster(): environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES environment_variables.environment_variables_dict['policy'] = 'instance_idle' with patch('boto3.client') as mock_client: - mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))] + mock_client.return_value.describe_instances.side_effect = [ + mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))] instance_idle = InstanceIdle() response = instance_idle.run() assert len(response) == 0 @@ -111,12 +114,13 @@ def test_instance_idle__dryrun_no_active_instance(): environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO environment_variables.environment_variables_dict['policy'] = 'instance_idle' with patch('boto3.client') as mock_client: - mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))] + mock_client.return_value.describe_instances.side_effect = [ + mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))] mock_client.return_value.get_metric_data.side_effect = [ MockCloudWatchMetric(metrics=[5, 4, 8, 10]).create_metric(), MockCloudWatchMetric(metrics=[5000, 2000, 4000, 8000]).create_metric(), MockCloudWatchMetric(metrics=[1000, 200, 500]).create_metric() - ] + ] instance_idle = InstanceIdle() response = instance_idle.run() assert len(response) == 0 @@ -157,7 +161,8 @@ def test_instance_idle__skips_delete(): environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO environment_variables.environment_variables_dict['policy'] = 'instance_idle' with patch('boto3.client') as mock_client: - mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))] + mock_client.return_value.describe_instances.side_effect = [ + mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))] mock_client.return_value.get_metric_data.side_effect = [ MockCloudWatchMetric(metrics=[0, 1, 0, 0.1]).create_metric(), MockCloudWatchMetric(metrics=[50, 20, 5, 10]).create_metric(), @@ -168,7 +173,6 @@ def test_instance_idle__skips_delete(): assert len(response) == 0 - def test_instance_idle__set_counter_zero(): """ This method tests unused_nat_gateway to set days counter to 0 @@ -179,7 +183,8 @@ def test_instance_idle__set_counter_zero(): environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES environment_variables.environment_variables_dict['policy'] = 'instance_idle' with patch('boto3.client') as mock_client: - mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))] + mock_client.return_value.describe_instances.side_effect = [ + mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))] mock_client.return_value.get_metric_data.side_effect = [ MockCloudWatchMetric(metrics=[0, 1, 0, 0.1]).create_metric(), MockCloudWatchMetric(metrics=[50, 20, 5, 10]).create_metric(),