Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added the ec2_run policy #703

Merged
merged 1 commit into from
Dec 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ This tool support the following policies:

* Real time Openshift Cluster cost, User cost
* [ec2_idle](cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb.
* [ec2_run](cloud_governance/policy/aws/ec2_run.py): running ec2.
* [ec2_run](cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2.
* [ebs_unattached](cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status.
* [ebs_in_use](cloud_governance/policy/aws/ebs_in_use.py): in use volumes.
* [tag_resources](cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags
Expand Down
8 changes: 8 additions & 0 deletions cloud_governance/common/clouds/aws/ec2/ec2_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -656,3 +656,11 @@ def describe_tags(self, **kwargs):
ec2_service_tags = self.ec2_client.describe_tags(NextToken=ec2_service_tags.get('NextToken'), **kwargs)
tags_list.extend(ec2_service_tags.get('Tags', []))
return tags_list

def get_running_instance(self):
"""
This method returns the EC2 running instances
:return:
:rtype:
"""
return self.get_ec2_instance_list(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])
13 changes: 8 additions & 5 deletions cloud_governance/common/clouds/aws/s3/s3_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,8 @@ def get_last_objects(self, bucket: str, logs_bucket_key: str = '', policy: str =
date_key = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y/%m/%d")
key_prefix = f'{logs_bucket_key}/{policy}/{date_key}'
objs = self.__s3_client.list_objects_v2(Bucket=bucket, Prefix=key_prefix)['Contents']
except:
except Exception as err:
print(err)
return None
get_last_modified_key = lambda obj: int(obj['LastModified'].strftime('%s'))
full_path = [obj['Key'] for obj in sorted(objs, key=get_last_modified_key)][-1]
Expand Down Expand Up @@ -294,26 +295,28 @@ def find_bucket(self, bucket_name: str):
return True
return False

def __get_s3_latest_policy_file(self, policy: str):
def __get_s3_latest_policy_file(self, policy: str, key_prefix: str = ''):
"""
This method return latest policy logs
@param policy:
@return:
"""
return self.get_last_objects(bucket=self.__bucket,
logs_bucket_key=f'{self.__logs_bucket_key}/{self.__region}',
key_prefix=key_prefix,
policy=policy)

def get_last_s3_policy_content(self, policy: str = '', file_name: str = '', s3_file_path: str = None):
def get_last_s3_policy_content(self, policy: str = '', file_name: str = '', s3_file_path: str = None,
key_prefix: str = ''):
"""
This method return last policy content
@return:
"""
with tempfile.TemporaryDirectory() as temp_local_directory:
local_file = temp_local_directory + '/' + file_name + '.gz'
if not s3_file_path:
if self.__get_s3_latest_policy_file(policy=policy):
s3_file_path = self.__get_s3_latest_policy_file(policy=policy)
if self.__get_s3_latest_policy_file(policy=policy, key_prefix=key_prefix):
s3_file_path = self.__get_s3_latest_policy_file(policy=policy, key_prefix=key_prefix)
self.download_file(bucket=self.__bucket,
key=str(s3_file_path),
download_file=file_name + '.gz',
Expand Down
Empty file.
Empty file.
145 changes: 145 additions & 0 deletions cloud_governance/common/helpers/aws/aws_cleanup_operations.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
import datetime

import boto3

from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations
from cloud_governance.common.logger.init_logger import logger


class AWSCleanUpOperations(AbstractCleanUpOperations):

def __init__(self):
super().__init__()
self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
self.__s3operations = S3Operations(region_name=self._region)
self._ec2_client = boto3.client('ec2', region_name=self._region)
self._s3_client = boto3.client('s3')
self._iam_client = boto3.client('iam')

def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
"""
This method returns the tag value from the tags
:param tags:
:type tags:
:param tag_name:
:type tag_name:
:return:
:rtype:
"""
if tags:
for tag in tags:
if tag.get('Key').strip().lower() == tag_name.lower():
return tag.get('Value').strip()
return ''

def get_clean_up_days_count(self, tags: list):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems it also updates the cleanup days value ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, It returns the clean_up days count. We have another method which updates the days count.
Check here.

"""
This method returns the cleanup days count
:param tags:
:type tags:
:return:
:rtype:
"""
last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount')
if not last_used_day:
return 1
else:
date, days = last_used_day.split('@')
if date != str(self.CURRENT_DATE):
return int(days) + 1
return 1 if int(days) == 0 else int(days)

def _delete_resource(self, resource_id: str):
"""
This method deletes the resource by verifying the policy
:param resource_id:
:type resource_id:
:return:
:rtype:
"""
action = "deleted"
try:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should replace to case

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder, does python have the switch case? 🤔

if self._policy == 's3_inactive':
self._s3_client.delete_bucket(Bucket=resource_id)
elif self._policy == 'empty_roles':
self._iam_client.delete_role(RoleName=resource_id)
elif self._policy == 'ebs_unattached':
self._ec2_client.delete_volume(VolumeId=resource_id)
elif self._policy == 'ip_unattached':
self._ec2_client.release_address(AllocationId=resource_id)
elif self._policy == 'unused_nat_gateway':
self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id)
elif self._policy == 'zombie_snapshots':
self._ec2_client.delete_snapshot(SnapshotId=resource_id)
elif self._policy == 'ec2_run':
self._ec2_client.stop_instances(InstanceIds=[resource_id])
action = "Stopped"
logger.info(f'{self._policy} {action}: {resource_id}')
except Exception as err:
logger.info(f'Exception raised: {err}: {resource_id}')

def __remove_tag_key_aws(self, tags: list):
"""
This method returns the tags that does not contain key startswith aws:
:param tags:
:type tags:
:return:
:rtype:
"""
custom_tags = []
for tag in tags:
if not tag.get('Key').lower().startswith('aws'):
custom_tags.append(tag)
return custom_tags

def __update_tag_value(self, tags: list, tag_name: str, tag_value: str):
"""
This method updates the tag_value
@param tags:
@param tag_name:
@param tag_value:
@return:
"""
if self._dry_run == "yes":
tag_value = 0
tag_value = f'{self.CURRENT_DATE}@{tag_value}'
found = False
if tags:
for tag in tags:
if tag.get('Key') == tag_name:
if tag.get('Value').split("@")[0] != self.CURRENT_DATE:
tag['Value'] = tag_value
else:
if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1:
tag['Value'] = tag_value
found = True
if not found:
tags.append({'Key': tag_name, 'Value': tag_value})
tags = self.__remove_tag_key_aws(tags=tags)
return tags

def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list, force_tag_update: str = ''):
"""
This method updates the resource tags
:param force_tag_update:
:type force_tag_update:
:param tags:
:type tags:
:param cleanup_days:
:type cleanup_days:
:param resource_id:
:type resource_id:
:return:
:rtype:
"""
tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
try:
if self._policy == 's3_inactive':
self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})
elif self._policy == 'empty_roles':
self._iam_client.tag_role(RoleName=resource_id, Tags=tags)
elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', 'ec2_run'):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about new policies that we will add in the future line s3_age, do you need add it also here ?
its very risky to tide to specific policy name !!!

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I know. I was thinking about how I change the process. As of now, we will stick to it later we can replace it.

self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
except Exception as err:
logger.info(f'Exception raised: {err}: {resource_id}')
111 changes: 111 additions & 0 deletions cloud_governance/common/helpers/cleanup_operations.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Union

from cloud_governance.main.environment_variables import environment_variables


class AbstractCleanUpOperations(ABC):

DAYS_TO_NOTIFY_ADMINS = 2
DAYS_TO_TRIGGER_RESOURCE_MAIL = 4
DAILY_HOURS = 24
CURRENT_DATE = datetime.utcnow().date().__str__()

def __init__(self):
self._environment_variables_dict = environment_variables.environment_variables_dict
self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION')
self._dry_run = self._environment_variables_dict.get('dry_run')
self._policy = self._environment_variables_dict.get('policy')
self._force_delete = self._environment_variables_dict.get('FORCE_DELETE')
self._resource_id = self._environment_variables_dict.get('RESOURCE_ID')

@abstractmethod
def get_clean_up_days_count(self, tags: Union[list, dict]):
"""
This method returns the cleanup days count
:param tags:
:type tags:
:return:
:rtype:
"""
raise NotImplementedError("This method is Not yet implemented")

@abstractmethod
def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str):
"""
This method returns the tag_value from the tags
:param tags:
:type tags:
:param tag_name:
:type tag_name:
:return:
:rtype:
"""
raise NotImplementedError("This method is Not yet implemented")

def get_skip_policy_value(self, tags: Union[list, dict]) -> str:
"""
This method returns the skip value
:param tags:
:type tags:
:return:
:rtype:
"""
policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Policy').strip()
if not policy_value:
policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Skip').strip()
if policy_value:
return policy_value.replace('_', '').replace('-', '').upper()
return 'NA'

@abstractmethod
def _delete_resource(self, resource_id: str):
"""
This method deletes the resource
:param resource_id:
:type resource_id:
:return:
:rtype:
"""
raise NotImplementedError("This method is Not yet implemented")

@abstractmethod
def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list):
"""
This method updates the resource tags
:param resource_id:
:type resource_id:
:param cleanup_days:
:type cleanup_days:
:param tags:
:type tags:
:return:
:rtype:
"""
raise NotImplementedError("This method is Not yet implemented")

def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int,
days_to_delete_resource: int = None, **kwargs):
"""
This method verify and delete the resource by calculating the days
:return:
:rtype:
"""
if self._resource_id == resource_id and self._force_delete and self._dry_run == 'no':
self._delete_resource(resource_id=resource_id)
return True
if not days_to_delete_resource:
days_to_delete_resource = self._days_to_take_action
cleanup_resources = False
if clean_up_days >= self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL:
if clean_up_days == self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL:
kwargs['delta_cost'] = kwargs.get('extra_purse')
# @Todo, If it require add email alert. May In future will add the email alert.
else:
if clean_up_days >= days_to_delete_resource:
if self._dry_run == 'no':
if self.get_skip_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP'):
self._delete_resource(resource_id=resource_id)
cleanup_resources = True
return cleanup_resources
10 changes: 10 additions & 0 deletions cloud_governance/common/helpers/json_datetime_encoder.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import json
import datetime


class JsonDateTimeEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
# Serialize datetime objects to ISO 8601 format
return obj.isoformat()
return super(JsonDateTimeEncoder, self).default(obj)
41 changes: 41 additions & 0 deletions cloud_governance/main/aws_main_operations.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import os

from cloud_governance.main.environment_variables import environment_variables
from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner


class AWSMainOperations:

def __init__(self):
self.__environment_variables_dict = environment_variables.environment_variables_dict
self.__policy = self.__environment_variables_dict.get('policy', '')
self.__policy_runner = PolicyRunner()

def __get_policies(self) -> dict:
"""
This method gets the aws policies
:return:
:rtype:
"""
policies = {}
policies_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'policy', 'aws')
for (dirpath, dirnames, filenames) in os.walk(policies_path):
immediate_parent = dirpath.split("/")[-1]
for filename in filenames:
if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')):
policies.setdefault(immediate_parent, []).append(os.path.splitext(filename)[0])
return policies

def run(self):
"""
This method run the AWS Policy operations
:return:
:rtype:
"""
policies_list = self.__get_policies()
for policy_type, policies in policies_list.items():
# @Todo support for all the aws policies, currently supports ec2_run as urgent requirement
if self.__policy in policies and self.__policy == "ec2_run":
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why you ask for specific policy self.__policy == "ec2_run"

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As I specified in the new ADR, this is only for ec2_run. So this should work against only ec2_run. I'll activate other policies later. Its risky to activate all policies without testing.

self.__policy_runner.run(source=policy_type)
return True
return False
Loading