-
Notifications
You must be signed in to change notification settings - Fork 15
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Added the ec2_run policy #703
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,145 @@ | ||
import datetime | ||
|
||
import boto3 | ||
|
||
from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations | ||
from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations | ||
from cloud_governance.common.logger.init_logger import logger | ||
|
||
|
||
class AWSCleanUpOperations(AbstractCleanUpOperations): | ||
|
||
def __init__(self): | ||
super().__init__() | ||
self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') | ||
self.__s3operations = S3Operations(region_name=self._region) | ||
self._ec2_client = boto3.client('ec2', region_name=self._region) | ||
self._s3_client = boto3.client('s3') | ||
self._iam_client = boto3.client('iam') | ||
|
||
def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str: | ||
""" | ||
This method returns the tag value from the tags | ||
:param tags: | ||
:type tags: | ||
:param tag_name: | ||
:type tag_name: | ||
:return: | ||
:rtype: | ||
""" | ||
if tags: | ||
for tag in tags: | ||
if tag.get('Key').strip().lower() == tag_name.lower(): | ||
return tag.get('Value').strip() | ||
return '' | ||
|
||
def get_clean_up_days_count(self, tags: list): | ||
""" | ||
This method returns the cleanup days count | ||
:param tags: | ||
:type tags: | ||
:return: | ||
:rtype: | ||
""" | ||
last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount') | ||
if not last_used_day: | ||
return 1 | ||
else: | ||
date, days = last_used_day.split('@') | ||
if date != str(self.CURRENT_DATE): | ||
return int(days) + 1 | ||
return 1 if int(days) == 0 else int(days) | ||
|
||
def _delete_resource(self, resource_id: str): | ||
""" | ||
This method deletes the resource by verifying the policy | ||
:param resource_id: | ||
:type resource_id: | ||
:return: | ||
:rtype: | ||
""" | ||
action = "deleted" | ||
try: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should replace to case There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder, does python have the switch case? 🤔 |
||
if self._policy == 's3_inactive': | ||
self._s3_client.delete_bucket(Bucket=resource_id) | ||
elif self._policy == 'empty_roles': | ||
self._iam_client.delete_role(RoleName=resource_id) | ||
elif self._policy == 'ebs_unattached': | ||
self._ec2_client.delete_volume(VolumeId=resource_id) | ||
elif self._policy == 'ip_unattached': | ||
self._ec2_client.release_address(AllocationId=resource_id) | ||
elif self._policy == 'unused_nat_gateway': | ||
self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id) | ||
elif self._policy == 'zombie_snapshots': | ||
self._ec2_client.delete_snapshot(SnapshotId=resource_id) | ||
elif self._policy == 'ec2_run': | ||
self._ec2_client.stop_instances(InstanceIds=[resource_id]) | ||
action = "Stopped" | ||
logger.info(f'{self._policy} {action}: {resource_id}') | ||
except Exception as err: | ||
logger.info(f'Exception raised: {err}: {resource_id}') | ||
|
||
def __remove_tag_key_aws(self, tags: list): | ||
""" | ||
This method returns the tags that does not contain key startswith aws: | ||
:param tags: | ||
:type tags: | ||
:return: | ||
:rtype: | ||
""" | ||
custom_tags = [] | ||
for tag in tags: | ||
if not tag.get('Key').lower().startswith('aws'): | ||
custom_tags.append(tag) | ||
return custom_tags | ||
|
||
def __update_tag_value(self, tags: list, tag_name: str, tag_value: str): | ||
""" | ||
This method updates the tag_value | ||
@param tags: | ||
@param tag_name: | ||
@param tag_value: | ||
@return: | ||
""" | ||
if self._dry_run == "yes": | ||
tag_value = 0 | ||
tag_value = f'{self.CURRENT_DATE}@{tag_value}' | ||
found = False | ||
if tags: | ||
for tag in tags: | ||
if tag.get('Key') == tag_name: | ||
if tag.get('Value').split("@")[0] != self.CURRENT_DATE: | ||
tag['Value'] = tag_value | ||
else: | ||
if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1: | ||
tag['Value'] = tag_value | ||
found = True | ||
if not found: | ||
tags.append({'Key': tag_name, 'Value': tag_value}) | ||
tags = self.__remove_tag_key_aws(tags=tags) | ||
return tags | ||
|
||
def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list, force_tag_update: str = ''): | ||
""" | ||
This method updates the resource tags | ||
:param force_tag_update: | ||
:type force_tag_update: | ||
:param tags: | ||
:type tags: | ||
:param cleanup_days: | ||
:type cleanup_days: | ||
:param resource_id: | ||
:type resource_id: | ||
:return: | ||
:rtype: | ||
""" | ||
tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) | ||
try: | ||
if self._policy == 's3_inactive': | ||
self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags}) | ||
elif self._policy == 'empty_roles': | ||
self._iam_client.tag_role(RoleName=resource_id, Tags=tags) | ||
elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', 'ec2_run'): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What about new policies that we will add in the future line s3_age, do you need add it also here ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, I know. I was thinking about how I change the process. As of now, we will stick to it later we can replace it. |
||
self._ec2_client.create_tags(Resources=[resource_id], Tags=tags) | ||
except Exception as err: | ||
logger.info(f'Exception raised: {err}: {resource_id}') |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,111 @@ | ||
from abc import ABC, abstractmethod | ||
from datetime import datetime | ||
from typing import Union | ||
|
||
from cloud_governance.main.environment_variables import environment_variables | ||
|
||
|
||
class AbstractCleanUpOperations(ABC): | ||
|
||
DAYS_TO_NOTIFY_ADMINS = 2 | ||
DAYS_TO_TRIGGER_RESOURCE_MAIL = 4 | ||
DAILY_HOURS = 24 | ||
CURRENT_DATE = datetime.utcnow().date().__str__() | ||
|
||
def __init__(self): | ||
self._environment_variables_dict = environment_variables.environment_variables_dict | ||
self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION') | ||
self._dry_run = self._environment_variables_dict.get('dry_run') | ||
self._policy = self._environment_variables_dict.get('policy') | ||
self._force_delete = self._environment_variables_dict.get('FORCE_DELETE') | ||
self._resource_id = self._environment_variables_dict.get('RESOURCE_ID') | ||
|
||
@abstractmethod | ||
def get_clean_up_days_count(self, tags: Union[list, dict]): | ||
""" | ||
This method returns the cleanup days count | ||
:param tags: | ||
:type tags: | ||
:return: | ||
:rtype: | ||
""" | ||
raise NotImplementedError("This method is Not yet implemented") | ||
|
||
@abstractmethod | ||
def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str): | ||
""" | ||
This method returns the tag_value from the tags | ||
:param tags: | ||
:type tags: | ||
:param tag_name: | ||
:type tag_name: | ||
:return: | ||
:rtype: | ||
""" | ||
raise NotImplementedError("This method is Not yet implemented") | ||
|
||
def get_skip_policy_value(self, tags: Union[list, dict]) -> str: | ||
""" | ||
This method returns the skip value | ||
:param tags: | ||
:type tags: | ||
:return: | ||
:rtype: | ||
""" | ||
policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Policy').strip() | ||
if not policy_value: | ||
policy_value = self.get_tag_name_from_tags(tags=tags, tag_name='Skip').strip() | ||
if policy_value: | ||
return policy_value.replace('_', '').replace('-', '').upper() | ||
return 'NA' | ||
|
||
@abstractmethod | ||
def _delete_resource(self, resource_id: str): | ||
""" | ||
This method deletes the resource | ||
:param resource_id: | ||
:type resource_id: | ||
:return: | ||
:rtype: | ||
""" | ||
raise NotImplementedError("This method is Not yet implemented") | ||
|
||
@abstractmethod | ||
def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: list): | ||
""" | ||
This method updates the resource tags | ||
:param resource_id: | ||
:type resource_id: | ||
:param cleanup_days: | ||
:type cleanup_days: | ||
:param tags: | ||
:type tags: | ||
:return: | ||
:rtype: | ||
""" | ||
raise NotImplementedError("This method is Not yet implemented") | ||
|
||
def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int, | ||
days_to_delete_resource: int = None, **kwargs): | ||
""" | ||
This method verify and delete the resource by calculating the days | ||
:return: | ||
:rtype: | ||
""" | ||
if self._resource_id == resource_id and self._force_delete and self._dry_run == 'no': | ||
self._delete_resource(resource_id=resource_id) | ||
return True | ||
if not days_to_delete_resource: | ||
days_to_delete_resource = self._days_to_take_action | ||
cleanup_resources = False | ||
if clean_up_days >= self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL: | ||
if clean_up_days == self._days_to_take_action - self.DAYS_TO_TRIGGER_RESOURCE_MAIL: | ||
kwargs['delta_cost'] = kwargs.get('extra_purse') | ||
# @Todo, If it require add email alert. May In future will add the email alert. | ||
else: | ||
if clean_up_days >= days_to_delete_resource: | ||
if self._dry_run == 'no': | ||
if self.get_skip_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP'): | ||
self._delete_resource(resource_id=resource_id) | ||
cleanup_resources = True | ||
return cleanup_resources |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
import json | ||
import datetime | ||
|
||
|
||
class JsonDateTimeEncoder(json.JSONEncoder): | ||
def default(self, obj): | ||
if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)): | ||
# Serialize datetime objects to ISO 8601 format | ||
return obj.isoformat() | ||
return super(JsonDateTimeEncoder, self).default(obj) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
import os | ||
|
||
from cloud_governance.main.environment_variables import environment_variables | ||
from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner | ||
|
||
|
||
class AWSMainOperations: | ||
|
||
def __init__(self): | ||
self.__environment_variables_dict = environment_variables.environment_variables_dict | ||
self.__policy = self.__environment_variables_dict.get('policy', '') | ||
self.__policy_runner = PolicyRunner() | ||
|
||
def __get_policies(self) -> dict: | ||
""" | ||
This method gets the aws policies | ||
:return: | ||
:rtype: | ||
""" | ||
policies = {} | ||
policies_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'policy', 'aws') | ||
for (dirpath, dirnames, filenames) in os.walk(policies_path): | ||
immediate_parent = dirpath.split("/")[-1] | ||
for filename in filenames: | ||
if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')): | ||
policies.setdefault(immediate_parent, []).append(os.path.splitext(filename)[0]) | ||
return policies | ||
|
||
def run(self): | ||
""" | ||
This method run the AWS Policy operations | ||
:return: | ||
:rtype: | ||
""" | ||
policies_list = self.__get_policies() | ||
for policy_type, policies in policies_list.items(): | ||
# @Todo support for all the aws policies, currently supports ec2_run as urgent requirement | ||
if self.__policy in policies and self.__policy == "ec2_run": | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why you ask for specific policy self.__policy == "ec2_run" There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As I specified in the new ADR, this is only for ec2_run. So this should work against only ec2_run. I'll activate other policies later. Its risky to activate all policies without testing. |
||
self.__policy_runner.run(source=policy_type) | ||
return True | ||
return False |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Seems it also updates the cleanup days value ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No, It returns the clean_up days count. We have another method which updates the days count.
Check here.