Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Re arranged the order of policies #716

Merged
merged 1 commit into from
Jan 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

This file was deleted.

52 changes: 40 additions & 12 deletions cloud_governance/policy/aws/cleanup/instance_run.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,63 @@
import datetime
from datetime import datetime

from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations
from cloud_governance.policy.abstract_policies.cleanup.abstractinstance_run import AbstractInstanceRun
from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations


class InstanceRun(AbstractInstanceRun, AWSPolicyOperations):
class InstanceRun(AWSPolicyOperations):

INSTANCE_TYPES_ES_INDEX = "cloud-governance-instance-types"
RESOURCE_ACTION = "Stopped"

def __init__(self):
super().__init__()

def _upload_instance_type_count_to_elastic_search(self):
"""
This method uploads the instance type count to elasticsearch
:return:
:rtype:
"""
instance_types = self._update_instance_type_count()
account = self.account
current_day = datetime.utcnow()
es_instance_types_data = []
for region, instance_types in instance_types.items():
for instance_type, instance_type_count in instance_types.items():
es_instance_types_data.append({
'instance_type': instance_type,
'instance_count': instance_type_count,
'timestamp': current_day,
'region': region,
'account': account,
'PublicCloud': self._cloud_name,
'index_id': f'{instance_type}-{self._cloud_name.lower()}-{account.lower()}-{region}-{str(current_day.date())}'
})
self._es_upload.es_upload_data(items=es_instance_types_data, es_index=self.INSTANCE_TYPES_ES_INDEX,
set_index='index_id')

def _update_instance_type_count(self):
"""
This method returns the instance types count by region
:return: { region: { instance_type: count } }
:rtype: dict
"""
instance_types = {}
resources = self._get_al_instances()
resources = self._get_all_instances()
for instance in resources:
instance_type = instance.get('InstanceType')
instance_types.setdefault(self._region, {}).update(
{instance_type: instance_types.get(instance_type, 0) + 1}
{instance_type: instance_types.get(self._region).get(instance_type, 0) + 1}
)
return instance_types

def _instance_run(self):
def run_policy_operations(self):
"""
This method returns the running instances
:return:
:rtype:
"""
instances = self._get_al_instances()
self._upload_instance_type_count_to_elastic_search()
instances = self._get_all_instances()
running_instances_data = []
for instance in instances:
tags = instance.get('Tags', [])
Expand All @@ -40,18 +67,19 @@ def _instance_run(self):
cleanup_result = self.verify_and_delete_resource(
resource_id=instance.get('InstanceId'), tags=tags,
clean_up_days=cleanup_days)
resource_data = self._get_es_data_schema_format(
resource_data = self._get_es_schema(
resource_id=instance.get('InstanceId'),
skip_policy=self.get_skip_policy_value(tags=tags),
user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
launch_time=instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"),
instance_type=instance.get('InstanceType'),
instance_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped',
resource_type=instance.get('InstanceType'),
resource_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped',
resource_action=self.RESOURCE_ACTION,
running_days=running_days, cleanup_days=cleanup_days,
dry_run=self._dry_run,
name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
region=self._region, cleanup_result=str(cleanup_result),
cloud_name=self._cloud_name
cloud_name=self._cloud_name,
)
if self._force_delete and self._dry_run == 'no':
resource_data.update({'ForceDeleted': str(self._force_delete)})
Expand Down
48 changes: 39 additions & 9 deletions cloud_governance/policy/azure/cleanup/instance_run.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,47 @@
from cloud_governance.common.helpers.azure.azure_policy_operations import AzurePolicyOperations
from cloud_governance.policy.abstract_policies.cleanup.abstractinstance_run import AbstractInstanceRun
from datetime import datetime

from cloud_governance.policy.helpers.azure.azure_policy_operations import AzurePolicyOperations

class InstanceRun(AbstractInstanceRun, AzurePolicyOperations):

class InstanceRun(AzurePolicyOperations):

INSTANCE_TYPES_ES_INDEX = "cloud-governance-instance-types"
RESOURCE_ACTION = "Stopped"

def __init__(self):
super().__init__()

def _upload_instance_type_count_to_elastic_search(self):
"""
athiruma marked this conversation as resolved.
Show resolved Hide resolved
This method uploads the instance type count to elasticsearch
:return:
:rtype:
"""
instance_types = self._update_instance_type_count()
account = self.account
current_day = datetime.utcnow()
es_instance_types_data = []
for region, instance_types in instance_types.items():
for instance_type, instance_type_count in instance_types.items():
es_instance_types_data.append({
'instance_type': instance_type,
'instance_count': instance_type_count,
'timestamp': current_day,
'region': region,
'account': account,
'PublicCloud': self._cloud_name,
'index_id': f'{instance_type}-{self._cloud_name.lower()}-{account.lower()}-{region}-{str(current_day.date())}'
})
self._es_upload.es_upload_data(items=es_instance_types_data, es_index=self.INSTANCE_TYPES_ES_INDEX,
set_index='index_id')

def _update_instance_type_count(self):
"""
This method returns the instance type count by region
:return: { region : {instance_type: instance_count} }
:rtype: dict
"""
resources = self._get_al_instances()
resources = self._get_all_instances()
instance_types = {}
for resource in resources:
vm_type = resource.hardware_profile.vm_size
Expand Down Expand Up @@ -43,13 +71,14 @@ def __get_instance_status(self, resource_id: str, vm_name: str):
status = 'Unknown Status'
return status

def _instance_run(self):
def run_policy_operations(self):
"""
This method returns the running vms in the AAzure cloud and stops based on the action
:return:
:rtype:
"""
vms_list = self._get_al_instances()
self._upload_instance_type_count_to_elastic_search()
vms_list = self._get_all_instances()
running_vms = []
for vm in vms_list:
status = self.__get_instance_status(resource_id=vm.id, vm_name=vm.name)
Expand All @@ -59,16 +88,17 @@ def _instance_run(self):
cleanup_days = self.get_clean_up_days_count(tags=tags)
cleanup_result = self.verify_and_delete_resource(resource_id=vm.id, tags=tags,
clean_up_days=cleanup_days)
resource_data = self._get_es_data_schema_format(
resource_data = self._get_es_schema(
resource_id=vm.name,
skip_policy=self.get_skip_policy_value(tags=tags),
user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
launch_time=vm.time_created,
instance_type=vm.hardware_profile.vm_size,
instance_state=status if cleanup_result else 'Vm Stopped',
resource_type=vm.hardware_profile.vm_size,
resource_state=status if cleanup_result else 'Vm Stopped',
running_days=running_days, cleanup_days=cleanup_days,
dry_run=self._dry_run,
name=vm.name,
resource_action=self.RESOURCE_ACTION,
region=vm.location, cleanup_result=str(cleanup_result),
cloud_name=self._cloud_name
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from datetime import datetime
from typing import Union

from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
from cloud_governance.main.environment_variables import environment_variables


Expand All @@ -20,6 +21,7 @@ def __init__(self):
self._policy = self._environment_variables_dict.get('policy')
self._force_delete = self._environment_variables_dict.get('FORCE_DELETE')
self._resource_id = self._environment_variables_dict.get('RESOURCE_ID')
self._es_upload = ElasticUpload()

def calculate_days(self, create_date: Union[datetime, str]):
"""
Expand Down Expand Up @@ -149,10 +151,59 @@ def _update_tag_value(self, tags: Union[list, dict], tag_name: str, tag_value: s
raise NotImplementedError("This method is Not yet implemented")

@abstractmethod
def _get_al_instances(self):
def _get_all_instances(self):
"""
This method returns all the instances
:return:
:rtype:
"""
raise NotImplementedError("This method not yet implemented")

def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str,
name: str, region: str, cleanup_result: str, resource_action: str, cloud_name: str,
resource_state: str, resource_type: str, **kwargs):
"""
This method returns the es schema data format
:return:
:rtype:
"""
current_date = datetime.utcnow().date()
resource_data = {
'ResourceId': resource_id,
'User': user,
'SkipPolicy': skip_policy,
'ResourceType': resource_type,
'ResourceState': resource_state,
'CleanUpDays': cleanup_days,
'DryRun': dry_run,
'Name': name,
'RegionName': region,
f'Resource{resource_action}': cleanup_result,
'PublicCloud': cloud_name,
'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}'
}
if kwargs.get('launch_time'):
resource_data.update({'LaunchTime': kwargs.get('launch_time')})
if kwargs.get('running_days'):
resource_data.update({'RunningDays': kwargs.get('running_days')})
if kwargs.get('volume_size'):
resource_data.update({'VolumeSize': kwargs.get('volume_size')})

return resource_data

@abstractmethod
def run_policy_operations(self):
"""
This method returns policy output
:return:
:rtype:
"""
pass

def run(self):
"""
This method starts the policy operations
:return:
:rtype:
"""
return self.run_policy_operations()
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
from cloud_governance.common.helpers.abstract_policy_operations import AbstractPolicyOperations
from cloud_governance.policy.helpers.abstract_policy_operations import AbstractPolicyOperations
from cloud_governance.common.logger.init_logger import logger


Expand All @@ -19,7 +19,6 @@ def __init__(self):
self._s3_client = boto3.client('s3')
self._iam_client = boto3.client('iam')


def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
athiruma marked this conversation as resolved.
Show resolved Hide resolved
"""
This method returns the tag value from the tags
Expand Down Expand Up @@ -131,11 +130,14 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
except Exception as err:
logger.info(f'Exception raised: {err}: {resource_id}')

def _get_al_instances(self):
def _get_all_instances(self):
"""
This method updates the instance type count to the elasticsearch
:return:
:rtype:
"""
instances = self._ec2_operations.get_ec2_instance_list()
return instances

def run_policy_operations(self):
raise NotImplementedError("This method needs to be implemented")
Loading
Loading