Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add the unit price to policies #830

Merged
merged 1 commit into from
Sep 6, 2024
Merged

Add the unit price to policies #830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions cloud_governance/common/clouds/aws/price/resources_pricing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,49 @@ def get_rds_price(self, region_name: str, instance_type: str):
]
unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict)
return round(unit_price, DEFAULT_ROUND_DIGITS)

def get_snapshot_unit_price(self, region_name: str):
"""
This method returns the unit price of Ebs Snapshot
:param region_name:
:return:
"""
service_code = 'AmazonEC2'
filter_dict = [
{"Field": "regionCode", "Value": region_name, "Type": "TERM_MATCH"},
{"Field": "productFamily", "Value": "Storage Snapshot", "Type": "TERM_MATCH"},
{"Field": "snapshotarchivefeetype", "Value": "SnapshotArchiveStorage", "Type": "TERM_MATCH"},
]
unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict)
return round(unit_price, DEFAULT_ROUND_DIGITS)

def get_ec2_price(self, region_name: str, instance_type: str, operating_system: str = None):
"""
This method returns the unit price of Ec2 Price
:param operating_system:
:param region_name:
:param instance_type:
:return:
"""
if not operating_system:
operating_system = 'Linux/UNIX'
os_types = {'Linux/UNIX': 'Linux',
'Red Hat Enterprise Linux': 'RHEL',
'SUSE Linux': 'SUSE',
'Ubuntu Pro Linux': 'Ubuntu Pro',
'Windows': 'Windows',
'Red Hat Enterprise Linux with High Availability': 'Red Hat Enterprise Linux with HA'}
operating_system_value = "NA"
for os_type in os_types.keys():
if os_type.lower() == operating_system.lower():
operating_system_value = os_types[os_type]
service_code = 'AmazonEC2'
filter_dict = [
{"Field": "regionCode", "Value": region_name, "Type": "TERM_MATCH"},
{"Field": "tenancy", "Value": "shared", "Type": "TERM_MATCH"},
{"Field": "operatingSystem", "Value": f"{operating_system_value}", "Type": "TERM_MATCH"},
{"Field": "instanceType", "Value": f"{instance_type}", "Type": "TERM_MATCH"},
{"Field": "capacitystatus", "Value": "Used", "Type": "TERM_MATCH"},
]
unit_price = self._aws_pricing.get_service_pricing(service_code, filter_dict)
return round(unit_price, DEFAULT_ROUND_DIGITS)
7 changes: 5 additions & 2 deletions cloud_governance/policy/aws/cleanup/instance_idle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

from cloud_governance.common.utils.configs import INSTANCE_IDLE_DAYS
from cloud_governance.common.utils.utils import Utils
from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations
Expand Down Expand Up @@ -38,6 +37,9 @@ def run_policy_operations(self):
self.get_skip_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP') and \
self.verify_instance_idle(resource_id=instance_id):
cleanup_days = self.get_clean_up_days_count(tags=tags)
unit_price = self._resource_pricing.get_ec2_price(region_name=self._region,
instance_type=instance.get('InstanceType'),
operating_system=instance.get('PlatformDetails'))
cleanup_result = self.verify_and_delete_resource(resource_id=instance_id, tags=tags,
clean_up_days=cleanup_days)
resource_data = self._get_es_schema(
Expand All @@ -52,7 +54,8 @@ def run_policy_operations(self):
name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
resource_action=self.RESOURCE_ACTION,
region=self._region, cleanup_result=str(cleanup_result),
cloud_name=self._cloud_name
cloud_name=self._cloud_name,
unit_price=unit_price
)
if self._force_delete and self._dry_run == 'no':
resource_data.update({'ForceDeleted': str(self._force_delete)})
Expand Down
6 changes: 5 additions & 1 deletion cloud_governance/policy/aws/cleanup/instance_run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@


class InstanceRun(AWSPolicyOperations):

INSTANCE_TYPES_ES_INDEX = "cloud-governance-instance-types"
RESOURCE_ACTION = "Stopped"

Expand Down Expand Up @@ -62,8 +61,12 @@ def run_policy_operations(self):
for instance in instances:
tags = instance.get('Tags', [])
cleanup_result = False

if instance.get('State', {}).get('Name') == 'running':
running_days = self.calculate_days(instance.get('LaunchTime'))
unit_price = self._resource_pricing.get_ec2_price(region_name=self._region,
instance_type=instance.get('InstanceType'),
operating_system=instance.get('PlatformDetails'))
if self._shutdown_period:
cleanup_days = self.get_clean_up_days_count(tags=tags)
cleanup_result = self.verify_and_delete_resource(
Expand All @@ -84,6 +87,7 @@ def run_policy_operations(self):
name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
region=self._region, cleanup_result=str(cleanup_result),
cloud_name=self._cloud_name,
unit_price=unit_price
)
if self._shutdown_period and self._force_delete and self._dry_run == 'no':
resource_data.update({'ForceDeleted': str(self._force_delete)})
Expand Down
7 changes: 5 additions & 2 deletions cloud_governance/policy/aws/zombie_snapshots.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from cloud_governance.common.utils.configs import HOURS_IN_MONTH
from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations


Expand Down Expand Up @@ -32,6 +33,7 @@ def run(self):
This method returns all the zombie snapshots and delete after x days
@return:
"""
monthly_price = self._resource_pricing.get_snapshot_unit_price(region_name=self._region)
snapshots = self._ec2_operations.get_snapshots()
zombie_snapshots = []
for snapshot in snapshots:
Expand All @@ -45,7 +47,7 @@ def run(self):
cleanup_days = self.get_clean_up_days_count(tags=tags)
cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags,
clean_up_days=cleanup_days)
unit_price = 0
unit_price = (monthly_price / HOURS_IN_MONTH) * float(snapshot.get('VolumeSize'))
resource_data = self._get_es_schema(resource_id=resource_id,
user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
skip_policy=self.get_skip_policy_value(tags=tags),
Expand All @@ -57,7 +59,8 @@ def run(self):
cloud_name=self._cloud_name,
resource_type='Snapshot',
volume_size=f"{snapshot.get('VolumeSize')} GB",
unit_price=unit_price, resource_state='Backup' if not cleanup_result else "Deleted"
unit_price=unit_price,
resource_state='Backup' if not cleanup_result else "Deleted"
)
zombie_snapshots.append(resource_data)
if not cleanup_result:
Expand Down
Empty file added tests/integration/cloud_governance/common/clouds/aws/price/__init__.py
View file
Open in desktop
Empty file.
14 changes: 14 additions & 0 deletions tests/integration/cloud_governance/common/clouds/aws/price/test_price.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
from cloud_governance.common.clouds.aws.price.price import AWSPrice


def test_get_service_pricing():
aws_price = AWSPrice()
service_code = 'AmazonEC2'
filter_list = [
{"Field": "regionCode", "Value": 'ap-south-1', "Type": "TERM_MATCH"},
{"Field": "tenancy", "Value": "shared", "Type": "TERM_MATCH"},
{"Field": "operatingSystem", "Value": "Linux", "Type": "TERM_MATCH"},
{"Field": "instanceType", "Value": "t2.micro", "Type": "TERM_MATCH"},
{"Field": "capacitystatus", "Value": "Used", "Type": "TERM_MATCH"},
]
assert aws_price.get_service_pricing(service_code, filter_list) > 0
31 changes: 18 additions & 13 deletions tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_idle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

from datetime import datetime, timedelta
from typing import Union
from unittest.mock import patch
Expand Down Expand Up @@ -41,7 +40,8 @@ def mock_describe_instances(*args, **kwargs):
'InstanceId': 'i-1234567890abcdef0',
'State': {'Name': 'running'},
'LaunchTime': kwargs.get('LaunchTime', datetime.utcnow()),
'Tags': kwargs.get('Tags', [])
'Tags': kwargs.get('Tags', []),
'PlatformDetails': 'Linux/UNIX'
# Change the launch time here
}
]
Expand Down Expand Up @@ -75,11 +75,13 @@ def test_instance_idle__check_not_idle():
environment_variables.environment_variables_dict['dry_run'] = 'yes'
environment_variables.environment_variables_dict['policy'] = 'instance_idle'
with patch('boto3.client') as mock_client:
mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.get_metric_data.side_effect = [MockCloudWatchMetric(metrics=[5, 4, 8, 10]).create_metric(),
MockCloudWatchMetric(metrics=[5000, 2000, 4000, 8000]).create_metric(),
MockCloudWatchMetric(metrics=[1000, 200, 500]).create_metric()
]
mock_client.return_value.describe_instances.side_effect = [
mock_describe_instances(LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.get_metric_data.side_effect = [
MockCloudWatchMetric(metrics=[5, 4, 8, 10]).create_metric(),
MockCloudWatchMetric(metrics=[5000, 2000, 4000, 8000]).create_metric(),
MockCloudWatchMetric(metrics=[1000, 200, 500]).create_metric()
]
instance_idle = InstanceIdle()
response = instance_idle.run()
assert len(response) == 0
Expand All @@ -95,7 +97,8 @@ def test_instance_idle__skip_cluster():
environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES
environment_variables.environment_variables_dict['policy'] = 'instance_idle'
with patch('boto3.client') as mock_client:
mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.describe_instances.side_effect = [
mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))]
instance_idle = InstanceIdle()
response = instance_idle.run()
assert len(response) == 0
Expand All @@ -111,12 +114,13 @@ def test_instance_idle__dryrun_no_active_instance():
environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO
environment_variables.environment_variables_dict['policy'] = 'instance_idle'
with patch('boto3.client') as mock_client:
mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.describe_instances.side_effect = [
mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.get_metric_data.side_effect = [
MockCloudWatchMetric(metrics=[5, 4, 8, 10]).create_metric(),
MockCloudWatchMetric(metrics=[5000, 2000, 4000, 8000]).create_metric(),
MockCloudWatchMetric(metrics=[1000, 200, 500]).create_metric()
]
]
instance_idle = InstanceIdle()
response = instance_idle.run()
assert len(response) == 0
Expand Down Expand Up @@ -157,7 +161,8 @@ def test_instance_idle__skips_delete():
environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO
environment_variables.environment_variables_dict['policy'] = 'instance_idle'
with patch('boto3.client') as mock_client:
mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.describe_instances.side_effect = [
mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.get_metric_data.side_effect = [
MockCloudWatchMetric(metrics=[0, 1, 0, 0.1]).create_metric(),
MockCloudWatchMetric(metrics=[50, 20, 5, 10]).create_metric(),
Expand All @@ -168,7 +173,6 @@ def test_instance_idle__skips_delete():
assert len(response) == 0



def test_instance_idle__set_counter_zero():
"""
This method tests unused_nat_gateway to set days counter to 0
Expand All @@ -179,7 +183,8 @@ def test_instance_idle__set_counter_zero():
environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES
environment_variables.environment_variables_dict['policy'] = 'instance_idle'
with patch('boto3.client') as mock_client:
mock_client.return_value.describe_instances.side_effect = [mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.describe_instances.side_effect = [
mock_describe_instances(Tags=tags, LaunchTime=datetime.utcnow() - timedelta(days=8))]
mock_client.return_value.get_metric_data.side_effect = [
MockCloudWatchMetric(metrics=[0, 1, 0, 0.1]).create_metric(),
MockCloudWatchMetric(metrics=[50, 20, 5, 10]).create_metric(),
Expand Down