diff --git a/README.md b/README.md index 29943d41..18371d74 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ This tool support the following policies: * Real time Openshift Cluster cost, User cost * [ec2_idle](cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb. -* [ec2_run](cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2. +* [instance_run](cloud_governance/policy/aws/cleanup/instance_run.py): running ec2. * [ebs_unattached](cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status. * [ebs_in_use](cloud_governance/policy/aws/ebs_in_use.py): in use volumes. * [tag_resources](cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags @@ -86,7 +86,7 @@ sudo podman pull quay.io/ebattat/cloud-governance (mandatory)AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY ##### Policy name: -(mandatory)policy=ec2_idle / ec2_run / ebs_unattached / ebs_in_use / tag_cluster_resource / zombie_cluster_resource / tag_ec2_resource +(mandatory)policy=ec2_idle / instance_run / ebs_unattached / ebs_in_use / tag_cluster_resource / zombie_cluster_resource / tag_ec2_resource ##### Policy logs output (mandatory)policy_output=s3://redhat-cloud-governance/logs @@ -133,8 +133,8 @@ GOOGLE_APPLICATION_CREDENTIALS=$pwd/service_account.json # policy=ec2_idle sudo podman run --rm --name cloud-governance -e policy="ec2_idle" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" -# policy=ec2_run -sudo podman run --rm --name cloud-governance -e policy="ec2_run" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +# policy=instance_run +sudo podman run --rm --name cloud-governance -e policy="instance_run" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" # select policy ['ec2_stop', 's3_inactive', 'empty_roles', 'ip_unattached', 'unused_nat_gateway', 'zombie_snapshots'] sudo podman run --rm --name cloud-governance -e policy="policy" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e log_level="INFO" "quay.io/ebattat/cloud-governance" diff --git a/cloud_governance/common/clouds/azure/compute/compute_operations.py b/cloud_governance/common/clouds/azure/compute/compute_operations.py index 3ca52a26..dee906a0 100644 --- a/cloud_governance/common/clouds/azure/compute/compute_operations.py +++ b/cloud_governance/common/clouds/azure/compute/compute_operations.py @@ -31,9 +31,9 @@ def get_all_instances(self) -> [VirtualMachine]: instances_list: [VirtualMachine] = self._item_paged_iterator(item_paged_object=instances_paged_object) return instances_list - def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine: + def get_instance_statuses(self, resource_id: str, vm_name: str) -> dict: """ - This method returns the virtual machine data by taking the id + This method returns the virtual machine instance status :param vm_name: :type vm_name: :param resource_id: @@ -42,6 +42,33 @@ def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine: :rtype: """ resource_group_name = self._get_resource_group_name_from_resource_id(resource_id=resource_id) - virtual_machine = self.__compute_client.virtual_machines.get(resource_group_name=resource_group_name, - vm_name=vm_name) - return virtual_machine + virtual_machine = self.__compute_client.virtual_machines.instance_view(resource_group_name=resource_group_name, + vm_name=vm_name) + return virtual_machine.as_dict() + + def get_id_dict_data(self, resource_id: str): + """ + This method generates the vm id dictionary + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + pairs = resource_id.split('/')[1:] + key_pairs = {pairs[i].lower(): pairs[i + 1] for i in range(0, len(pairs), 2)} + return key_pairs + + def stop_vm(self, resource_id: str): + """ + This method stops the vm + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + id_key_pairs = self.get_id_dict_data(resource_id) + resource_group_name = id_key_pairs.get('resourcegroups') + vm_name = id_key_pairs.get('virtualmachines') + status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name, + vm_name=vm_name) + return status.done() diff --git a/cloud_governance/common/elasticsearch/elasticsearch_operations.py b/cloud_governance/common/elasticsearch/elasticsearch_operations.py index bf723d5a..5281dd30 100644 --- a/cloud_governance/common/elasticsearch/elasticsearch_operations.py +++ b/cloud_governance/common/elasticsearch/elasticsearch_operations.py @@ -144,6 +144,8 @@ def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc' # Upload data to elastic search server if 'account' not in map(str.lower, data.keys()): data['account'] = self.__account + if data.get('index-id'): + kwargs['id'] = data.get('index-id') try: if isinstance(data, dict): # JSON Object self.__es.index(index=index, doc_type=doc_type, body=data, **kwargs) @@ -310,6 +312,8 @@ def upload_data_in_bulk(self, data_items: list, index: str, **kwargs): for item in bulk_items: if kwargs.get('id'): item['_id'] = item.get(kwargs.get('id')) + if item.get('index-id'): + item['_id'] = item.get('index-id') if not item.get('timestamp'): if 'CurrentDate' in item: item['timestamp'] = datetime.strptime(item.get('CurrentDate'), "%Y-%m-%d") diff --git a/cloud_governance/common/helpers/cleanup_operations.py b/cloud_governance/common/helpers/abstract_policy_operations.py similarity index 72% rename from cloud_governance/common/helpers/cleanup_operations.py rename to cloud_governance/common/helpers/abstract_policy_operations.py index aa8ac152..461ccc5d 100644 --- a/cloud_governance/common/helpers/cleanup_operations.py +++ b/cloud_governance/common/helpers/abstract_policy_operations.py @@ -5,7 +5,7 @@ from cloud_governance.main.environment_variables import environment_variables -class AbstractCleanUpOperations(ABC): +class AbstractPolicyOperations(ABC): DAYS_TO_NOTIFY_ADMINS = 2 DAYS_TO_TRIGGER_RESOURCE_MAIL = 4 @@ -14,13 +14,27 @@ class AbstractCleanUpOperations(ABC): def __init__(self): self._environment_variables_dict = environment_variables.environment_variables_dict + self.account = self._environment_variables_dict.get('account') self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION') self._dry_run = self._environment_variables_dict.get('dry_run') self._policy = self._environment_variables_dict.get('policy') self._force_delete = self._environment_variables_dict.get('FORCE_DELETE') self._resource_id = self._environment_variables_dict.get('RESOURCE_ID') - @abstractmethod + def calculate_days(self, create_date: Union[datetime, str]): + """ + This method returns the days + :param create_date: + :type create_date: + :return: + :rtype: + """ + if isinstance(create_date, str): + create_date = datetime.strptime(create_date, "%Y-%M-%d") + today = datetime.utcnow().date() + days = today - create_date.date() + return days.days + def get_clean_up_days_count(self, tags: Union[list, dict]): """ This method returns the cleanup days count @@ -29,7 +43,16 @@ def get_clean_up_days_count(self, tags: Union[list, dict]): :return: :rtype: """ - raise NotImplementedError("This method is Not yet implemented") + if self._dry_run == 'yes': + return 0 + last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount') + if not last_used_day: + return 1 + else: + date, days = last_used_day.split('@') + if date != str(self.CURRENT_DATE): + return int(days) + 1 + return 1 if int(days) == 0 else int(days) @abstractmethod def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str): @@ -85,7 +108,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag """ raise NotImplementedError("This method is Not yet implemented") - def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int, + def verify_and_delete_resource(self, resource_id: str, tags: Union[list, dict], clean_up_days: int, days_to_delete_resource: int = None, **kwargs): """ This method verify and delete the resource by calculating the days @@ -109,3 +132,27 @@ def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days self._delete_resource(resource_id=resource_id) cleanup_resources = True return cleanup_resources + + @abstractmethod + def _update_tag_value(self, tags: Union[list, dict], tag_name: str, tag_value: str): + """ + This method returns the updated tag_list by adding the tag_name and tag_value to the tags + :param tags: + :type tags: + :param tag_name: + :type tag_name: + :param tag_value: + :type tag_value: + :return: + :rtype: + """ + raise NotImplementedError("This method is Not yet implemented") + + @abstractmethod + def _get_al_instances(self): + """ + This method returns all the instances + :return: + :rtype: + """ + raise NotImplementedError("This method not yet implemented") diff --git a/cloud_governance/common/helpers/aws/aws_cleanup_operations.py b/cloud_governance/common/helpers/aws/aws_policy_operations.py similarity index 82% rename from cloud_governance/common/helpers/aws/aws_cleanup_operations.py rename to cloud_governance/common/helpers/aws/aws_policy_operations.py index 4539ade5..c8492f09 100644 --- a/cloud_governance/common/helpers/aws/aws_cleanup_operations.py +++ b/cloud_governance/common/helpers/aws/aws_policy_operations.py @@ -1,22 +1,25 @@ -import datetime import boto3 +from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations -from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations +from cloud_governance.common.helpers.abstract_policy_operations import AbstractPolicyOperations from cloud_governance.common.logger.init_logger import logger -class AWSCleanUpOperations(AbstractCleanUpOperations): +class AWSPolicyOperations(AbstractPolicyOperations): def __init__(self): super().__init__() self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') + self._cloud_name = 'AWS' self.__s3operations = S3Operations(region_name=self._region) self._ec2_client = boto3.client('ec2', region_name=self._region) + self._ec2_operations = EC2Operations(region=self._region) self._s3_client = boto3.client('s3') self._iam_client = boto3.client('iam') + def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str: """ This method returns the tag value from the tags @@ -33,23 +36,6 @@ def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str: return tag.get('Value').strip() return '' - def get_clean_up_days_count(self, tags: list): - """ - This method returns the cleanup days count - :param tags: - :type tags: - :return: - :rtype: - """ - last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount') - if not last_used_day: - return 1 - else: - date, days = last_used_day.split('@') - if date != str(self.CURRENT_DATE): - return int(days) + 1 - return 1 if int(days) == 0 else int(days) - def _delete_resource(self, resource_id: str): """ This method deletes the resource by verifying the policy @@ -72,7 +58,7 @@ def _delete_resource(self, resource_id: str): self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id) elif self._policy == 'zombie_snapshots': self._ec2_client.delete_snapshot(SnapshotId=resource_id) - elif self._policy == 'ec2_run': + elif self._policy == 'instance_run': self._ec2_client.stop_instances(InstanceIds=[resource_id]) action = "Stopped" logger.info(f'{self._policy} {action}: {resource_id}') @@ -93,9 +79,9 @@ def __remove_tag_key_aws(self, tags: list): custom_tags.append(tag) return custom_tags - def __update_tag_value(self, tags: list, tag_name: str, tag_value: str): + def _update_tag_value(self, tags: list, tag_name: str, tag_value: str): """ - This method updates the tag_value + This method returns the updated tag_list by adding the tag_name and tag_value to the tags @param tags: @param tag_name: @param tag_value: @@ -133,13 +119,23 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag :return: :rtype: """ - tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) + tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) try: if self._policy == 's3_inactive': self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags}) elif self._policy == 'empty_roles': self._iam_client.tag_role(RoleName=resource_id, Tags=tags) - elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', 'ec2_run'): + elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', + 'instance_run'): self._ec2_client.create_tags(Resources=[resource_id], Tags=tags) except Exception as err: logger.info(f'Exception raised: {err}: {resource_id}') + + def _get_al_instances(self): + """ + This method updates the instance type count to the elasticsearch + :return: + :rtype: + """ + instances = self._ec2_operations.get_ec2_instance_list() + return instances diff --git a/cloud_governance/common/helpers/azure/__init__.py b/cloud_governance/common/helpers/azure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/helpers/azure/azure_policy_operations.py b/cloud_governance/common/helpers/azure/azure_policy_operations.py new file mode 100644 index 00000000..f13c41a1 --- /dev/null +++ b/cloud_governance/common/helpers/azure/azure_policy_operations.py @@ -0,0 +1,91 @@ + +from cloud_governance.common.clouds.azure.compute.compute_operations import ComputeOperations +from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations +from cloud_governance.common.helpers.abstract_policy_operations import AbstractPolicyOperations +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.common.utils.utils import Utils + + +class AzurePolicyOperations(AbstractPolicyOperations): + + def __init__(self): + self._cloud_name = 'Azure' + self.compute_operations = ComputeOperations() + self.resource_group_operations = ResourceGroupOperations() + super().__init__() + + def get_tag_name_from_tags(self, tags: dict, tag_name: str): + """ + This method returns the tag value by the tag_name + :param tags: + :type tags: + :param tag_name: + :type tag_name: + :return: + :rtype: + """ + if tags: + for key, value in tags.items(): + if Utils.equal_ignore_case(key, tag_name): + return value + return '' + + def _delete_resource(self, resource_id: str): + """ + This method deletes the + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + action = "deleted" + try: + if self._policy == 'instance_run': + action = "Stopped" + self.compute_operations.stop_vm(resource_id=resource_id) + logger.info(f'{self._policy} {action}: {resource_id}') + except Exception as err: + logger.info(f'Exception raised: {err}: {resource_id}') + + def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict): + tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) + try: + if self._policy == 'instance_run': + self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags) + except Exception as err: + logger.info(f'Exception raised: {err}: {resource_id}') + + def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str): + """ + This method returns the updated tag_list by adding the tag_name and tag_value to the tags + @param tags: + @param tag_name: + @param tag_value: + @return: + """ + if self._dry_run == "yes": + tag_value = 0 + tag_value = f'{self.CURRENT_DATE}@{tag_value}' + found = False + updated_tags = {} + if tags: + for key, value in tags.items(): + if Utils.equal_ignore_case(key, tag_name): + if value.split("@")[0] != self.CURRENT_DATE: + updated_tags[key] = tag_value + else: + if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1: + updated_tags[key] = tag_value + found = True + tags.update(updated_tags) + if not found: + tags.update({tag_name: tag_value}) + return tags + + def _get_al_instances(self): + """ + This method returns the all instances list + :return: + :rtype: + """ + return self.compute_operations.get_all_instances() \ No newline at end of file diff --git a/cloud_governance/common/utils/utils.py b/cloud_governance/common/utils/utils.py new file mode 100644 index 00000000..09ddee7f --- /dev/null +++ b/cloud_governance/common/utils/utils.py @@ -0,0 +1,51 @@ + +import os + + +class Utils: + + def __init__(self): + pass + + @staticmethod + def get_cloud_policies(cloud_name: str, file_type: str = '.py', dir_dict: bool = False, + exclude_policies: list = None): + """ + This method returns the policies by cloud_name + :return: + :rtype: + """ + cloud_name = cloud_name.lower() + exclude_policies = [] if not exclude_policies else exclude_policies + policies_dict = {} + policies_list = [] + policies_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'policy', cloud_name) + for (dir_path, _, filenames) in os.walk(policies_path): + immediate_parent = dir_path.split("/")[-1] + for filename in filenames: + if not filename.startswith('__') and filename.endswith(file_type): + filename = os.path.splitext(filename)[0] + if filename not in exclude_policies: + if dir_dict: + policies_dict.setdefault(immediate_parent, []).append(filename) + else: + policies_list.append(filename) + return policies_dict if dir_dict else policies_list + + @staticmethod + def equal_ignore_case(str1: str, str2: str, *args): + """ + This method returns boolean by comparing equal in-case sensitive all strings + :param str1: + :type str1: + :param str2: + :type str2: + :param args: + :type args: + :return: + :rtype: + """ + equal = str1.lower() == str2.lower() + for val in args: + equal = str1.lower() == val.lower() and equal + return equal diff --git a/cloud_governance/main/aws_main_operations.py b/cloud_governance/main/aws_main_operations.py deleted file mode 100644 index 60823ff3..00000000 --- a/cloud_governance/main/aws_main_operations.py +++ /dev/null @@ -1,41 +0,0 @@ -import os - -from cloud_governance.main.environment_variables import environment_variables -from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner - - -class AWSMainOperations: - - def __init__(self): - self.__environment_variables_dict = environment_variables.environment_variables_dict - self.__policy = self.__environment_variables_dict.get('policy', '') - self.__policy_runner = PolicyRunner() - - def __get_policies(self) -> dict: - """ - This method gets the aws policies - :return: - :rtype: - """ - policies = {} - policies_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'policy', 'aws') - for (dirpath, dirnames, filenames) in os.walk(policies_path): - immediate_parent = dirpath.split("/")[-1] - for filename in filenames: - if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')): - policies.setdefault(immediate_parent, []).append(os.path.splitext(filename)[0]) - return policies - - def run(self): - """ - This method run the AWS Policy operations - :return: - :rtype: - """ - policies_list = self.__get_policies() - for policy_type, policies in policies_list.items(): - # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement - if self.__policy in policies and self.__policy == "ec2_run": - self.__policy_runner.run(source=policy_type) - return True - return False diff --git a/cloud_governance/main/main.py b/cloud_governance/main/main.py index 2f40633e..8b70852e 100644 --- a/cloud_governance/main/main.py +++ b/cloud_governance/main/main.py @@ -3,9 +3,8 @@ from ast import literal_eval # str to dict import boto3 # regions -from cloud_governance.cloud_resource_orchestration.monitor.cloud_monitor import CloudMonitor -from cloud_governance.main.aws_main_operations import AWSMainOperations from cloud_governance.main.main_common_operations import run_common_policies +from cloud_governance.main.main_oerations.main_operations import MainOperations from cloud_governance.main.run_cloud_resource_orchestration import run_cloud_resource_orchestration from cloud_governance.policy.policy_operations.aws.cost_expenditure.cost_report_policies import CostReportPolicies from cloud_governance.policy.policy_operations.azure.azure_policy_runner import AzurePolicyRunner @@ -206,10 +205,8 @@ def main(): es_index = environment_variables_dict.get('es_index', '') es_doc_type = environment_variables_dict.get('es_doc_type', '') bucket = environment_variables_dict.get('bucket', '') - response = False - if is_policy_aws(): - aws_main_operations = AWSMainOperations() - response = aws_main_operations.run() + main_operations = MainOperations() + response = main_operations.run() if not response: if environment_variables_dict.get('COMMON_POLICIES'): run_common_policies() diff --git a/cloud_governance/main/main_oerations/__init__.py b/cloud_governance/main/main_oerations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/main/main_oerations/main_operations.py b/cloud_governance/main/main_oerations/main_operations.py new file mode 100644 index 00000000..d287c2ce --- /dev/null +++ b/cloud_governance/main/main_oerations/main_operations.py @@ -0,0 +1,44 @@ + +from cloud_governance.common.utils.utils import Utils +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.policy_runners.azure.policy_runner import PolicyRunner as AzurePolicyRunner +from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner as AWSPolicyRunner + + +class MainOperations: + + def __init__(self): + self.utils = Utils() + self._environment_variables_dict = environment_variables.environment_variables_dict + self._policy = self._environment_variables_dict.get('policy', '') + self._public_cloud_name = self._environment_variables_dict.get('PUBLIC_CLOUD_NAME', '') + + def get_policy_runner(self): + """ + This method returns the cloud policy runner object + :return: + :rtype: + """ + policy_runner = None + if Utils.equal_ignore_case(self._public_cloud_name, 'AWS'): + policy_runner = AWSPolicyRunner() + else: + if Utils.equal_ignore_case(self._public_cloud_name, 'AZURE'): + policy_runner = AzurePolicyRunner() + + return policy_runner + + def run(self): + """ + This method run the AWS Policy operations + :return: + :rtype: + """ + policies_list = Utils.get_cloud_policies(cloud_name=self._public_cloud_name, dir_dict=True) + policy_runner = self.get_policy_runner() + for policy_type, policies in policies_list.items(): + # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement + if self._policy in policies and self._policy in "instance_run": + policy_runner.run(source=policy_type) + return True + return False diff --git a/cloud_governance/policy/abstract_policies/__init__.py b/cloud_governance/policy/abstract_policies/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/policy/abstract_policies/cleanup/abstractinstance_run.py b/cloud_governance/policy/abstract_policies/cleanup/abstractinstance_run.py new file mode 100644 index 00000000..78739ac8 --- /dev/null +++ b/cloud_governance/policy/abstract_policies/cleanup/abstractinstance_run.py @@ -0,0 +1,95 @@ +from abc import ABC, abstractmethod +from datetime import datetime + +from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload +from cloud_governance.main.environment_variables import environment_variables + + +class AbstractInstanceRun(ABC): + + INSTANCE_TYPES_ES_INDEX = 'cloud-governance-instance-types' + RESOURCE_ACTION = "Stopped" + + def __init__(self): + self.__es_upload = ElasticUpload() + self.__environment_variables_dict = environment_variables.environment_variables_dict + self.__account = self.__environment_variables_dict.get('account').upper().replace('OPENSHIFT-', '') + self.__cloud_name = self.__environment_variables_dict.get('PUBLIC_CLOUD_NAME') + super().__init__() + + def _upload_instance_type_count_to_elastic_search(self): + """ + This method uploads the data to elasticsearch + :return: + :rtype: + """ + instance_types = self._update_instance_type_count() + account = self.__account + current_day = datetime.utcnow() + es_instance_types_data = [] + for region, instance_types in instance_types.items(): + for instance_type, instance_type_count in instance_types.items(): + es_instance_types_data.append({ + 'instance_type': instance_type, + 'instance_count': instance_type_count, + 'timestamp': current_day, + 'region': region, + 'account': account, + 'PublicCloud': self.__cloud_name, + 'index_id': f'{instance_type}-{self.__cloud_name.lower()}-{account.lower()}-{region}-{str(current_day.date())}' + }) + self.__es_upload.es_upload_data(items=es_instance_types_data, es_index=self.INSTANCE_TYPES_ES_INDEX, + set_index='index_id') + + def _get_es_data_schema_format(self, resource_id: str, user: str, skip_policy: str, launch_time: datetime, + instance_type: str, instance_state: str, running_days: int, cleanup_days: int, + dry_run: str, name: str, region: str, cleanup_result: str, cloud_name: str): + """ + This method returns the schema of the es + :return: + :rtype: + """ + current_date = datetime.utcnow().date() + return { + 'ResourceId': resource_id, + 'User': user, + 'SkipPolicy': skip_policy, + 'LaunchTime': launch_time, + 'InstanceType': instance_type, + 'InstanceState': instance_state, + 'RunningDays': running_days, + 'CleanUpDays': cleanup_days, + 'DryRun': dry_run, + 'Name': name, + 'RegionName': region, + f'Resource{self.RESOURCE_ACTION}': cleanup_result, + 'PublicCloud': cloud_name, + 'index-id': f'{current_date}-{cloud_name.lower()}-{self.__account.lower()}-{region.lower()}-{resource_id}-{instance_state.lower()}' + } + + @abstractmethod + def _update_instance_type_count(self): + """ + This method updates the instance type count to the elasticsearch + :return: { region: { instance_type: count } } + :rtype: dict + """ + raise NotImplementedError("This method not yet implemented") + + @abstractmethod + def _instance_run(self): + """ + This method returns the running instances and upload to elastic_search + :return: + :rtype: + """ + raise NotImplementedError("This method not yet implemented") + + def run(self): + """ + This method starts the instance run operations + :return: + :rtype: + """ + self._upload_instance_type_count_to_elastic_search() + return self._instance_run() diff --git a/cloud_governance/policy/aws/cleanup/ec2_run.py b/cloud_governance/policy/aws/cleanup/ec2_run.py deleted file mode 100644 index 4b6ee19f..00000000 --- a/cloud_governance/policy/aws/cleanup/ec2_run.py +++ /dev/null @@ -1,86 +0,0 @@ -import datetime - -from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import \ - NonClusterZombiePolicy - - -class EC2Run(NonClusterZombiePolicy): - - RESOURCE_ACTION = "Stopped" - - def __init__(self): - super(EC2Run, self).__init__() - self.__es_index = 'cloud-governance-ec2-instance-types' - - def __update_instance_type_count(self, instances: list): - """ - This method updates the instance types count - :param instances: - :type instances: - :return: - :rtype: - """ - instance_types = {} - for instance in instances: - instance_type = instance.get('InstanceType') - instance_types[instance_type] = instance_types.get(instance_type, 0) + 1 - es_instance_types_data = [] - for key, value in instance_types.items(): - es_instance_types_data.append({ - 'instance_type': key, - 'instance_count': value, - 'timestamp': datetime.datetime.utcnow(), - 'region': self._region, - 'account': self._account.upper().replace('OPENSHIFT-', ''), - 'index_id': f'{key}-{self._account.lower()}-{self._region}-{str(datetime.datetime.utcnow().date())}' - }) - self._es_upload.es_upload_data(items=es_instance_types_data, es_index=self.__es_index, set_index='index_id') - - def __ec2_run(self): - """ - This method list the running instances and upload to elastic_search - :return: - :rtype: - """ - instances = self._ec2_operations.get_ec2_instance_list() - self.__update_instance_type_count(instances=instances) - running_instances_data = [] - for instance in instances: - tags = instance.get('Tags', []) - if instance.get('State', {}).get('Name') == 'running': - running_days = self._calculate_days(instance.get('LaunchTime')) - cleanup_days = self._aws_cleanup_policies.get_clean_up_days_count(tags=tags) - cleanup_result = self._aws_cleanup_policies.verify_and_delete_resource( - resource_id=instance.get('InstanceId'), tags=tags, - clean_up_days=cleanup_days) - resource_data = { - 'ResourceId': instance.get('InstanceId'), - 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'), - 'SkipPolicy': self._aws_cleanup_policies.get_skip_policy_value(tags=tags), - 'LaunchTime': instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), - 'InstanceType': instance.get('InstanceType'), - 'InstanceState': instance.get('State', {}).get('Name') if not cleanup_result else 'stopped', - 'StateTransitionReason': instance.get('StateTransitionReason'), - 'RunningDays': running_days, - 'CleanUpDays': cleanup_days, - 'DryRun': self._dry_run, - 'Name': self._get_tag_name_from_tags(tags=tags, tag_name='Name'), - 'RegionName': self._region, - f'Resource{self.RESOURCE_ACTION}': str(cleanup_result) - } - if self._force_delete and self._dry_run == 'no': - resource_data.update({'ForceDeleted': str(self._force_delete)}) - running_instances_data.append(resource_data) - else: - cleanup_days = 0 - self._aws_cleanup_policies.update_resource_day_count_tag(resource_id=instance.get('InstanceId'), - cleanup_days=cleanup_days, tags=tags) - - return running_instances_data - - def run(self): - """ - This method list all the running instances - @return: - """ - return self.__ec2_run() diff --git a/cloud_governance/policy/aws/cleanup/instance_run.py b/cloud_governance/policy/aws/cleanup/instance_run.py new file mode 100644 index 00000000..e74db2ff --- /dev/null +++ b/cloud_governance/policy/aws/cleanup/instance_run.py @@ -0,0 +1,64 @@ +import datetime + +from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations +from cloud_governance.policy.abstract_policies.cleanup.abstractinstance_run import AbstractInstanceRun + + +class InstanceRun(AbstractInstanceRun, AWSPolicyOperations): + + def __init__(self): + super().__init__() + + def _update_instance_type_count(self): + """ + This method returns the instance types count by region + :return: { region: { instance_type: count } } + :rtype: dict + """ + instance_types = {} + resources = self._get_al_instances() + for instance in resources: + instance_type = instance.get('InstanceType') + instance_types.setdefault(self._region, {}).update( + {instance_type: instance_types.get(instance_type, 0) + 1} + ) + return instance_types + + def _instance_run(self): + """ + This method returns the running instances + :return: + :rtype: + """ + instances = self._get_al_instances() + running_instances_data = [] + for instance in instances: + tags = instance.get('Tags', []) + if instance.get('State', {}).get('Name') == 'running': + running_days = self.calculate_days(instance.get('LaunchTime')) + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource( + resource_id=instance.get('InstanceId'), tags=tags, + clean_up_days=cleanup_days) + resource_data = self._get_es_data_schema_format( + resource_id=instance.get('InstanceId'), + skip_policy=self.get_skip_policy_value(tags=tags), + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + launch_time=instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), + instance_type=instance.get('InstanceType'), + instance_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped', + running_days=running_days, cleanup_days=cleanup_days, + dry_run=self._dry_run, + name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'), + region=self._region, cleanup_result=str(cleanup_result), + cloud_name=self._cloud_name + ) + if self._force_delete and self._dry_run == 'no': + resource_data.update({'ForceDeleted': str(self._force_delete)}) + running_instances_data.append(resource_data) + else: + cleanup_days = 0 + self.update_resource_day_count_tag(resource_id=instance.get('InstanceId'), cleanup_days=cleanup_days, + tags=tags) + + return running_instances_data diff --git a/cloud_governance/policy/azure/cleanup/__init__.py b/cloud_governance/policy/azure/cleanup/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/policy/azure/cleanup/instance_run.py b/cloud_governance/policy/azure/cleanup/instance_run.py new file mode 100644 index 00000000..6c03416e --- /dev/null +++ b/cloud_governance/policy/azure/cleanup/instance_run.py @@ -0,0 +1,81 @@ +from cloud_governance.common.helpers.azure.azure_policy_operations import AzurePolicyOperations +from cloud_governance.policy.abstract_policies.cleanup.abstractinstance_run import AbstractInstanceRun + + +class InstanceRun(AbstractInstanceRun, AzurePolicyOperations): + + def __init__(self): + super().__init__() + + def _update_instance_type_count(self): + """ + This method returns the instance type count by region + :return: { region : {instance_type: instance_count} } + :rtype: dict + """ + resources = self._get_al_instances() + instance_types = {} + for resource in resources: + vm_type = resource.hardware_profile.vm_size + region = resource.location + instance_types.setdefault(region, {}).update( + {vm_type: instance_types.get(region).get(vm_type, 0) + 1} + ) + return instance_types + + def __get_instance_status(self, resource_id: str, vm_name: str): + """ + This method returns the VM status of the Virtual Machine + :param resource_id: + :type resource_id: + :param vm_name: + :type vm_name: + :return: + :rtype: + """ + instance_statuses = self.compute_operations.get_instance_statuses(resource_id=resource_id, vm_name=vm_name) + statuses = instance_statuses.get('statuses', {}) + if len(statuses) >= 2: + status = statuses[1].get('display_status', '').lower() + elif len(statuses) == 1: + status = statuses[0].get('display_status', '').lower() + else: + status = 'Unknown Status' + return status + + def _instance_run(self): + """ + This method returns the running vms in the AAzure cloud and stops based on the action + :return: + :rtype: + """ + vms_list = self._get_al_instances() + running_vms = [] + for vm in vms_list: + status = self.__get_instance_status(resource_id=vm.id, vm_name=vm.name) + tags = vm.tags if vm.tags else {} + if 'running' in status: + running_days = self.calculate_days(vm.time_created) + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource(resource_id=vm.id, tags=tags, + clean_up_days=cleanup_days) + resource_data = self._get_es_data_schema_format( + resource_id=vm.name, + skip_policy=self.get_skip_policy_value(tags=tags), + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + launch_time=vm.time_created, + instance_type=vm.hardware_profile.vm_size, + instance_state=status if cleanup_result else 'Vm Stopped', + running_days=running_days, cleanup_days=cleanup_days, + dry_run=self._dry_run, + name=vm.name, + region=vm.location, cleanup_result=str(cleanup_result), + cloud_name=self._cloud_name + ) + if self._force_delete and self._dry_run == 'no': + resource_data.update({'ForceDeleted': str(self._force_delete)}) + running_vms.append(resource_data) + else: + cleanup_days = 0 + self.update_resource_day_count_tag(resource_id=vm.id, cleanup_days=cleanup_days, tags=tags) + return running_vms diff --git a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py index c043a9fe..9d8f7c19 100644 --- a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py +++ b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py @@ -10,7 +10,7 @@ from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations -from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations +from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations from cloud_governance.common.ldap.ldap_search import LdapSearch from cloud_governance.common.logger.init_logger import logger from cloud_governance.common.mails.mail_message import MailMessage @@ -27,7 +27,7 @@ class NonClusterZombiePolicy: DAILY_HOURS = 24 def __init__(self): - self._aws_cleanup_policies = AWSCleanUpOperations() + self._aws_cleanup_policies = AWSPolicyOperations() self.__environment_variables_dict = environment_variables.environment_variables_dict self._end_date = datetime.datetime.now() self._start_date = self._end_date - datetime.timedelta(days=self.DAYS_TO_DELETE_RESOURCE) diff --git a/cloud_governance/policy/policy_runners/aws/policy_runner.py b/cloud_governance/policy/policy_runners/aws/policy_runner.py index f09f77cb..3154c256 100644 --- a/cloud_governance/policy/policy_runners/aws/policy_runner.py +++ b/cloud_governance/policy/policy_runners/aws/policy_runner.py @@ -2,6 +2,8 @@ import importlib import inspect +from typing import Callable + from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations from cloud_governance.common.logger.init_logger import logger from cloud_governance.policy.policy_runners.aws.upload_s3 import UploadS3 @@ -13,40 +15,36 @@ class PolicyRunner(AbstractPolicyRunner): def __init__(self): super().__init__() - def run(self, source: str = "", upload: bool = True): + def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: bool): """ - This method run the AWS policies classes + This method executes the policy + :param policy_class_name: + :type policy_class_name: + :param run_policy: + :type run_policy: :param upload: :type upload: - :param source: - :type source: :return: :rtype: """ - source_policy = f"{source}.{self._policy}" if source else self._policy - logger.info(f'account={self._account}, policy={self._policy}, dry_run={self._dry_run}') - zombie_non_cluster_policy_module = importlib.import_module(f'cloud_governance.policy.aws.{source_policy}') - policy_result = [] ec2_operations = EC2Operations() upload_to_s3 = UploadS3() - for cls in inspect.getmembers(zombie_non_cluster_policy_module, inspect.isclass): - if self._policy.replace('_', '').replace('-', '') == cls[0].lower(): - active_regions = [self._region] - if self._run_active_regions: - active_regions = ec2_operations.get_active_regions() - logger.info("Running the policy in All AWS active regions") - for active_region in active_regions: - logger.info(f"Running the {self._policy} in Region: {active_region}") - self._environment_variables_dict['AWS_DEFAULT_REGION'] = active_region - response = cls[1]().run() - if isinstance(response, str): - logger.info(f'key: {cls[0]}, Response: {response}') - else: - policy_result.extend(response) - logger.info(f'key: {cls[0]}, count: {len(response)}, {response}') - if upload: - self._upload_elastic_search.upload(data=response) - upload_to_s3.upload(data=response) - if self._save_to_file_path: - self.write_to_file(data=policy_result) + active_regions = [self._region] + if self._run_active_regions: + active_regions = ec2_operations.get_active_regions() + logger.info("Running the policy in All AWS active regions") + for active_region in active_regions: + logger.info(f"Running the {self._policy} in Region: {active_region}") + self._environment_variables_dict['AWS_DEFAULT_REGION'] = active_region + response = run_policy().run() + if isinstance(response, str): + logger.info(f'key: {policy_class_name}, Response: {response}') + else: + policy_result.extend(response) + logger.info(f'key: {policy_class_name}, count: {len(response)}, {response}') + if upload: + self._upload_elastic_search.upload(data=response) + upload_to_s3.upload(data=response) + return policy_result + diff --git a/cloud_governance/policy/policy_runners/azure/__init__.py b/cloud_governance/policy/policy_runners/azure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/policy/policy_runners/azure/policy_runner.py b/cloud_governance/policy/policy_runners/azure/policy_runner.py new file mode 100644 index 00000000..f242b3c5 --- /dev/null +++ b/cloud_governance/policy/policy_runners/azure/policy_runner.py @@ -0,0 +1,31 @@ +from typing import Callable + +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.policy.policy_runners.common.abstract_policy_runner import AbstractPolicyRunner + + +class PolicyRunner(AbstractPolicyRunner): + + def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: bool = False): + """ + This method executes the policy + :param policy_class_name: + :type policy_class_name: + :param run_policy: + :type run_policy: + :param upload: + :type upload: + :return: + :rtype: + """ + policy_result = [] + response = run_policy().run() + if isinstance(response, str): + logger.info(response) + else: + policy_result.extend(response) + self._upload_elastic_search.upload(data=policy_result) + return policy_result + + def __init__(self): + super().__init__() diff --git a/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py b/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py index 63464c26..0d2a40e3 100644 --- a/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py +++ b/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py @@ -1,6 +1,8 @@ +import importlib +import inspect import os.path from abc import abstractmethod, ABC -from typing import Union +from typing import Union, Callable from cloud_governance.common.logger.init_logger import logger from cloud_governance.main.environment_variables import environment_variables @@ -18,12 +20,9 @@ def __init__(self): self._run_active_regions = self._environment_variables_dict.get('RUN_ACTIVE_REGIONS') self._upload_elastic_search = UploadElasticSearch() self._save_to_file_path = self._environment_variables_dict.get('SAVE_TO_FILE_PATH') + self._public_cloud_name = self._environment_variables_dict.get('PUBLIC_CLOUD_NAME', '') - @abstractmethod - def run(self): - raise NotImplementedError("This method is not yet implemented") - - def write_to_file(self, data: Union[list, dict]): + def write_to_file(self, data: Union[list, dict, str]): """ This method writes the data to file_path passed by the env SAVE_TO_FILE_PATH :param data: @@ -39,20 +38,59 @@ def write_to_file(self, data: Union[list, dict]): with open(file_name, 'w') as file: if isinstance(data, list): for item in data: - if not header_added: - keys = [str(val) for val in list(item.keys())] + ["\n"] - file.write(', '.join(keys)) - header_added = True - values = [str(val) for val in list(item.values())] + ["\n"] - file.write(', '.join(values)) + if isinstance(item, dict): + if not header_added: + keys = [str(val) for val in list(item.keys())] + ["\n"] + file.write(', '.join(keys)) + header_added = True + values = [str(val) for val in list(item.values())] + ["\n"] + file.write(', '.join(values)) + else: + file.write(f'{item}\n') else: if isinstance(data, dict): if not header_added: keys = [str(val) for val in list(data.keys())] + ["\n"] file.write(', '.join(keys)) - header_added = True values = [str(val) for val in list(data.values())] + ["\n"] file.write(', '.join(values)) + else: + file.write(data) + file.write('\n') logger.info(f"Written the data into the file_name: {file_name}") else: raise FileExistsError(f"FilePath not exists {self._save_to_file_path}") + + @abstractmethod + def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: bool): + """ + This method execute the policy + :return: + :rtype: + """ + raise NotImplementedError("This method is not yet implemented") + + def run(self, source: str = "", upload: bool = True): + """ + This method starts the method operations + :param source: + :type source: + :param upload: + :type upload: + :return: + :rtype: + """ + source_policy = f"{source}.{self._policy}" if source else self._policy + logger.info(f'CloudName={self._public_cloud_name}, account={self._account}, policy={self._policy}, dry_run={self._dry_run}') + policies_path = f'cloud_governance.policy.{self._public_cloud_name.lower()}.{source_policy}' + cloud_policies = importlib.import_module(policies_path) + policy_result = [] + + for cls in inspect.getmembers(cloud_policies, inspect.isclass): + if self._policy.replace('_', '').replace('-', '') == cls[0].lower(): + response = self.execute_policy(policy_class_name=cls[0], run_policy=cls[1], upload=upload) + policy_result.extend(response) + if self._save_to_file_path: + self.write_to_file(data=policy_result) + return policy_result + diff --git a/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py b/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py index a98e0ada..dfd56e72 100644 --- a/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py +++ b/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py @@ -30,8 +30,10 @@ def upload(self, data: Union[list, dict]): self._es_operations.upload_data_in_bulk(data_items=data.copy(), index=self._es_index) else: for policy_dict in data: - policy_dict['region_name'] = self._region - policy_dict['account'] = self._account + if 'RegionName' not in policy_dict: + policy_dict['RegionName'] = self._region + if 'account' not in policy_dict: + policy_dict['account'] = self._account self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index) logger.info(f'Uploaded the policy results to elasticsearch index: {self._es_index}') else: diff --git a/docs/source/index.md b/docs/source/index.md index 06c628fd..484ea5bc 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -11,7 +11,7 @@ This tool support the following policies: * Real time Openshift Cluster cost, User cost * [ec2_idle](../../cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb. -* [ec2_run](../../cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2. +* [ec2_run](../../cloud_governance/policy/aws/cleanup/instance_run.py): running ec2. * [ebs_unattached](../../cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status. * [ebs_in_use](../../cloud_governance/policy/aws/ebs_in_use.py): in use volumes. * [tag_resources](../../cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags diff --git a/jenkins/clouds/aws/daily/policies/run_policies.py b/jenkins/clouds/aws/daily/policies/run_policies.py index 737acbfa..ed62fb90 100644 --- a/jenkins/clouds/aws/daily/policies/run_policies.py +++ b/jenkins/clouds/aws/daily/policies/run_policies.py @@ -59,7 +59,7 @@ def get_policies(type: str = None): policies.remove('cost_explorer_payer_billings') policies.remove('spot_savings_analysis') policies.remove('optimize_resources_report') -policies.remove('ec2_run') +policies.remove('instance_run') es_index_env_var = f'-e es_index={ES_INDEX}' if ES_INDEX else '' @@ -84,7 +84,7 @@ def get_policies(type: str = None): elif policy in ('empty_roles', 's3_inactive') and region == 'us-east-1': os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e MANAGER_EMAIL_ALERT="False" -e account="PERF-DEPT" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e MANAGER_EMAIL_ALERT="False" -e account="PERF-SCALE" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF_SCALE}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") - # running policies dry_run=yes per every region ebs_in_use, ec2_run + # running policies dry_run=yes per every region ebs_in_use, instance_run else: if policy not in ('empty_roles', 's3_inactive'): os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-DEPT" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") @@ -92,9 +92,9 @@ def get_policies(type: str = None): os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-SCALE" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF_SCALE}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") # Run the EC2 run policy -os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-DEPT" -e policy="ec2_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e RUN_ACTIVE_REGIONS="True" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") -os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PSAP" -e MANAGER_EMAIL_ALERT="False" -e policy="ec2_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PSAP}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") -os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-SCALE" -e policy="ec2_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF_SCALE}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-DEPT" -e policy="instance_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e RUN_ACTIVE_REGIONS="True" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PSAP" -e MANAGER_EMAIL_ALERT="False" -e policy="instance_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e RUN_ACTIVE_REGIONS="True" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PSAP}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-SCALE" -e policy="instance_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e RUN_ACTIVE_REGIONS="True" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF_SCALE}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") diff --git a/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py b/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py index ae962bc9..f08a3369 100644 --- a/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py +++ b/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py @@ -1,4 +1,5 @@ import datetime +import json import boto3 import tempfile @@ -9,9 +10,10 @@ # walk around for moto DeprecationWarning import warnings -from cloud_governance.main.aws_main_operations import AWSMainOperations +from cloud_governance.common.helpers.json_datetime_encoder import JsonDateTimeEncoder from cloud_governance.main.environment_variables import environment_variables -from cloud_governance.policy.aws.cleanup.ec2_run import EC2Run +from cloud_governance.main.main_oerations.main_operations import MainOperations +from cloud_governance.policy.aws.cleanup.instance_run import AbstractInstanceRun with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -187,26 +189,27 @@ def test_folder_delete(): def test_get_s3_latest_policy_file(): region_name = 'us-east-1' bucket_name = 'test_s3_bucket' + policy = 'instance_run' s3_resource = boto3.resource('s3', region_name=region_name) s3_resource.create_bucket(Bucket=bucket_name) environment_variables.environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AWS' environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = policy environment_variables.environment_variables_dict['dry_run'] = 'yes' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name environment_variables.environment_variables_dict['policy_output'] = f's3://{bucket_name}/tests' - ec2_client = boto3.client('ec2', region_name='ap-south-1') + ec2_client = boto3.client('ec2', region_name=region_name) default_ami_id = 'ami-03cf127a' tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] - resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, + ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) - aws_main_operations = AWSMainOperations() - aws_main_operations.run() - current_date = datetime.datetime.now().date().__str__().replace('-', '/') - prefix = f'tests/{region_name}/ec2-run/{current_date}' + main_operations = MainOperations() + main_operations.run() s3_operations = S3Operations(region_name=region_name, bucket=bucket_name, logs_bucket_key='tests') - assert s3_operations._S3Operations__get_s3_latest_policy_file(policy='ec2-run', key_prefix=prefix) + current_date = datetime.datetime.now().date().__str__().replace('-', '/') + prefix = f'tests/{region_name}/instance-run/{current_date}' + assert s3_operations._S3Operations__get_s3_latest_policy_file(policy='instance-run', key_prefix=prefix) @mock_iam @@ -215,23 +218,24 @@ def test_get_s3_latest_policy_file(): def test_get_last_s3_policy_content(): region_name = 'us-east-1' bucket_name = 'test_s3_bucket' + policy = 'instance_run' s3_resource = boto3.resource('s3', region_name=region_name) s3_resource.create_bucket(Bucket=bucket_name) environment_variables.environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AWS' environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = policy environment_variables.environment_variables_dict['dry_run'] = 'yes' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name environment_variables.environment_variables_dict['policy_output'] = f's3://{bucket_name}/tests' - ec2_client = boto3.client('ec2', region_name='ap-south-1') + ec2_client = boto3.client('ec2', region_name=region_name) default_ami_id = 'ami-03cf127a' tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) - aws_main_operations = AWSMainOperations() - aws_main_operations.run() + main_operations = MainOperations() + main_operations.run() current_date = datetime.datetime.now().date().__str__().replace('-', '/') - key_prefix = f'tests/{region_name}/ec2-run/{current_date}' - s3_operations = S3Operations(region_name='us-east-1', bucket=bucket_name, logs_bucket_key='tests') - assert s3_operations.get_last_s3_policy_content(policy='ec2-run', file_name='resources.json', key_prefix=key_prefix) + key_prefix = f'tests/{region_name}/instance-run/{current_date}' + s3_operations = S3Operations(region_name=region_name, bucket=bucket_name, logs_bucket_key='tests') + assert s3_operations.get_last_s3_policy_content(policy='instance-run', file_name='resources.json', key_prefix=key_prefix) diff --git a/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py b/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py index e5006a90..82accba3 100644 --- a/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py +++ b/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py @@ -3,7 +3,7 @@ import boto3 from moto import mock_ec2, mock_s3, mock_iam -from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations +from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations from cloud_governance.main.environment_variables import environment_variables @@ -16,7 +16,7 @@ def test_get_tag_name_from_tags(): :return: :rtype: """ - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() tags = [{'Key': "Name", "Value": "Unittest"}] tag_value = aws_cleanup_operations.get_tag_name_from_tags(tags=tags, tag_name="Name") assert tag_value == "Unittest" @@ -32,10 +32,10 @@ def test_get_clean_up_days_count(): :rtype: """ environment_variables.environment_variables_dict['dry_run'] = 'yes' - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() tags = [{'Key': "Name", "Value": "Unittest"}] days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) - assert days_count == 1 + assert days_count == 0 @mock_ec2 @@ -48,11 +48,11 @@ def test_get_clean_up_days_count_already_exists(): :rtype: """ environment_variables.environment_variables_dict['dry_run'] = 'yes' - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() mock_date = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).date() tags = [{'Key': "Name", "Value": "Unittest"}, {'Key': "DaysCount", "Value": f'{mock_date}@1'}] days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) - assert days_count == 2 + assert days_count == 0 @mock_ec2 @@ -65,11 +65,11 @@ def test_get_clean_up_days_count_already_updated_today(): :rtype: """ environment_variables.environment_variables_dict['dry_run'] = 'yes' - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() mock_date = str(datetime.datetime.utcnow().date()) tags = [{'Key': "Name", "Value": "Unittest"}, {'Key': "DaysCount", "Value": f'{mock_date}@1'}] days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) - assert days_count == 1 + assert days_count == 0 @mock_ec2 @@ -81,7 +81,7 @@ def test_get_skip_policy_value_policy_tag(): :return: :rtype: """ - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() tags = [{'Key': "Name", "Value": "Unittest"}, {'Key': "Policy", "Value": "NotDelete"}] tag_value = aws_cleanup_operations.get_skip_policy_value(tags=tags) @@ -97,7 +97,7 @@ def test_get_skip_policy_value_skip_tag(): :return: :rtype: """ - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() tags = [{'Key': "Name", "Value": "Unittest"}, {'Key': "Skip", "Value": "NotDelete"}] tag_value = aws_cleanup_operations.get_skip_policy_value(tags=tags) @@ -113,7 +113,7 @@ def test_delete_resource(): :return: :rtype: """ - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') default_ami_id = 'ami-03cf127a' @@ -122,7 +122,7 @@ def test_delete_resource(): TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) if resource: resource_id = resource[0].get('InstanceId') - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() aws_cleanup_operations._delete_resource(resource_id=resource_id) assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -136,7 +136,7 @@ def test_update_resource_day_count_tag(): :return: :rtype: """ - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') default_ami_id = 'ami-03cf127a' @@ -146,7 +146,7 @@ def test_update_resource_day_count_tag(): []) if resource: resource_id = resource[0].get('InstanceId') - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() cleanup_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) aws_cleanup_operations.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags) instances = ec2_client.describe_instances()['Reservations'] @@ -163,7 +163,7 @@ def test_update_resource_day_count_tag_exists_tag(): :return: :rtype: """ - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' environment_variables.environment_variables_dict['dry_run'] = 'no' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -176,7 +176,7 @@ def test_update_resource_day_count_tag_exists_tag(): []) if resource: resource_id = resource[0].get('InstanceId') - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() cleanup_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) aws_cleanup_operations.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags) instances = ec2_client.describe_instances()['Reservations'] @@ -193,7 +193,7 @@ def test_update_resource_day_count_tag_updated_tag_today(): :return: :rtype: """ - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' environment_variables.environment_variables_dict['dry_run'] = 'no' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -206,7 +206,7 @@ def test_update_resource_day_count_tag_updated_tag_today(): []) if resource: resource_id = resource[0].get('InstanceId') - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() cleanup_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) aws_cleanup_operations.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags) instances = ec2_client.describe_instances()['Reservations'] diff --git a/tests/unittest/cloud_governance/common/helpers/test_cleanup_operations.py b/tests/unittest/cloud_governance/common/helpers/test_cleanup_operations.py index 85f516f9..eaf381cc 100644 --- a/tests/unittest/cloud_governance/common/helpers/test_cleanup_operations.py +++ b/tests/unittest/cloud_governance/common/helpers/test_cleanup_operations.py @@ -3,7 +3,7 @@ import boto3 from moto import mock_ec2, mock_s3, mock_iam -from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations +from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations from cloud_governance.main.environment_variables import environment_variables @@ -17,7 +17,7 @@ def test_verify_and_delete_resource_not_stopped(): :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') default_ami_id = 'ami-03cf127a' @@ -26,7 +26,7 @@ def test_verify_and_delete_resource_not_stopped(): TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) resource_id = resource[0].get('InstanceId') - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() clean_up_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) assert not aws_cleanup_operations.verify_and_delete_resource(resource_id=resource_id, tags=tags, clean_up_days=clean_up_days) @@ -43,7 +43,7 @@ def test_verify_and_delete_resource_stopped(): :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'no' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -55,7 +55,7 @@ def test_verify_and_delete_resource_stopped(): TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) resource_id = resource[0].get('InstanceId') - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() clean_up_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) assert aws_cleanup_operations.verify_and_delete_resource(resource_id=resource_id, tags=tags, clean_up_days=clean_up_days) @@ -72,7 +72,7 @@ def test_verify_and_delete_resource_skip(): :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'yes' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -85,7 +85,7 @@ def test_verify_and_delete_resource_skip(): TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) resource_id = resource[0].get('InstanceId') - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() clean_up_days = aws_cleanup_operations.get_clean_up_days_count(tags=tags) assert not aws_cleanup_operations.verify_and_delete_resource(resource_id=resource_id, tags=tags, clean_up_days=clean_up_days) diff --git a/tests/unittest/cloud_governance/common/utils/test_utils.py b/tests/unittest/cloud_governance/common/utils/test_utils.py index 769e6357..d57a0b62 100644 --- a/tests/unittest/cloud_governance/common/utils/test_utils.py +++ b/tests/unittest/cloud_governance/common/utils/test_utils.py @@ -19,3 +19,9 @@ def test_tag_aws_resources(): resource_ids.append(instance_id) expected_res = common_utils.tag_aws_resources(ec2_client.create_tags, tags=[{'Key': 'User', 'Value': 'test'}], resource_ids=resource_ids) assert expected_res == 2 + + +def test_get_cloud_policies(): + from cloud_governance.common.utils.utils import Utils + policies = Utils.get_cloud_policies(cloud_name='AWS', dir_dict=True) + assert 'instance_run' in policies['cleanup'] diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py similarity index 63% rename from tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py rename to tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py index 1f485956..723c676c 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py @@ -3,22 +3,22 @@ import boto3 from moto import mock_ec2, mock_s3, mock_iam -from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations +from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations from cloud_governance.main.environment_variables import environment_variables -from cloud_governance.policy.aws.cleanup.ec2_run import EC2Run +from cloud_governance.policy.aws.cleanup.instance_run import InstanceRun @mock_ec2 @mock_s3 @mock_iam -def test_ec2_run(): +def test_instance_run(): """ - This method tests ec2_run + This method tests instance_run :return: :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'no' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -27,8 +27,8 @@ def test_ec2_run(): resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) - ec2_run = EC2Run() - running_instances_data = ec2_run.run() + instance_run = InstanceRun() + running_instances_data = instance_run.run() assert running_instances_data[0].get('ResourceStopped') == 'False' assert running_instances_data[0].get('InstanceState') == 'running' @@ -36,14 +36,14 @@ def test_ec2_run(): @mock_ec2 @mock_s3 @mock_iam -def test_ec2_run_alert(): +def test_instance_run_alert(): """ - This method tests ec2_run + This method tests instance_run :return: :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'no' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -54,25 +54,12 @@ def test_ec2_run_alert(): 'Instances', []) resource = resource[0] - ec2_run = EC2Run() - running_instances_data = ec2_run.run() - assert running_instances_data == [ - { - 'ResourceId': resource.get('InstanceId'), - 'User': 'cloud-governance', - 'SkipPolicy': 'NA', - 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), - 'InstanceType': resource.get('InstanceType'), - 'InstanceState': 'running', - 'StateTransitionReason': resource.get('StateTransitionReason'), - 'RunningDays': 0, - 'CleanUpDays': 1, - 'DryRun': 'no', - 'Name': 'Unittest', - 'RegionName': 'ap-south-1', - 'ResourceStopped': 'False' - } - ] + instance_run = InstanceRun() + running_instances_data = instance_run.run() + assert len(running_instances_data) == 1 + assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') + assert running_instances_data[0]['DryRun'] == 'no' + assert running_instances_data[0]['ResourceStopped'] == 'False' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 @@ -80,14 +67,14 @@ def test_ec2_run_alert(): @mock_ec2 @mock_s3 @mock_iam -def test_ec2_run_alert_stopped(): +def test_instance_run_alert_stopped(): """ - This method tests ec2_run + This method tests instance_run :return: :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'no' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -100,39 +87,25 @@ def test_ec2_run_alert_stopped(): 'Instances', []) resource = resource[0] - ec2_run = EC2Run() - running_instances_data = ec2_run.run() - assert running_instances_data == [ - { - 'ResourceId': resource.get('InstanceId'), - 'User': 'cloud-governance', - 'SkipPolicy': 'NA', - 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), - 'InstanceType': resource.get('InstanceType'), - 'InstanceState': 'stopped', - 'StateTransitionReason': resource.get('StateTransitionReason'), - 'RunningDays': 0, - 'CleanUpDays': 4, - 'DryRun': 'no', - 'Name': 'Unittest', - 'RegionName': 'ap-south-1', - 'ResourceStopped': 'True' - } - ] + instance_run = InstanceRun() + running_instances_data = instance_run.run() + assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') + assert running_instances_data[0]['DryRun'] == 'no' + assert running_instances_data[0]['ResourceStopped'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @mock_ec2 @mock_s3 @mock_iam -def test_ec2_run_alert_skip(): +def test_instance_run_alert_skip(): """ - This method tests ec2_run + This method tests instance_run :return: :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'no' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -145,39 +118,25 @@ def test_ec2_run_alert_skip(): 'Instances', []) resource = resource[0] - ec2_run = EC2Run() - running_instances_data = ec2_run.run() - assert running_instances_data == [ - { - 'ResourceId': resource.get('InstanceId'), - 'User': 'cloud-governance', - 'SkipPolicy': 'NOTDELETE', - 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), - 'InstanceType': resource.get('InstanceType'), - 'InstanceState': 'running', - 'StateTransitionReason': resource.get('StateTransitionReason'), - 'RunningDays': 0, - 'CleanUpDays': 4, - 'DryRun': 'no', - 'Name': 'Unittest', - 'RegionName': 'ap-south-1', - 'ResourceStopped': 'False' - } - ] + instance_run = InstanceRun() + running_instances_data = instance_run.run() + assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') + assert running_instances_data[0]['DryRun'] == 'no' + assert running_instances_data[0]['ResourceStopped'] == 'False' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 @mock_ec2 @mock_s3 @mock_iam -def test_ec2_run_stop_reset(): +def test_instance_run_stop_reset(): """ - This method tests ec2_run + This method tests instance_run :return: :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'no' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -190,44 +149,30 @@ def test_ec2_run_stop_reset(): 'Instances', []) resource = resource[0] - ec2_run = EC2Run() - running_instances_data = ec2_run.run() - assert running_instances_data == [ - { - 'ResourceId': resource.get('InstanceId'), - 'User': 'cloud-governance', - 'SkipPolicy': 'NA', - 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), - 'InstanceType': resource.get('InstanceType'), - 'InstanceState': 'stopped', - 'StateTransitionReason': resource.get('StateTransitionReason'), - 'RunningDays': 0, - 'CleanUpDays': 4, - 'DryRun': 'no', - 'Name': 'Unittest', - 'RegionName': 'ap-south-1', - 'ResourceStopped': 'True' - } - ] + instance_run = InstanceRun() + running_instances_data = instance_run.run() + assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') + assert running_instances_data[0]['DryRun'] == 'no' + assert running_instances_data[0]['ResourceStopped'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 - ec2_run.run() + instance_run.run() instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations'] instances = instances[0]['Instances'][0] - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() assert aws_cleanup_operations.get_tag_name_from_tags(tags=instances.get('Tags'), tag_name='DaysCount').split('@')[-1] == '0' @mock_ec2 @mock_s3 @mock_iam -def test_ec2_run_stop_start(): +def test_instance_run_stop_start(): """ - This method tests ec2_run + This method tests instance_run :return: :rtype: """ environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 2 - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'no' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -240,33 +185,20 @@ def test_ec2_run_stop_start(): 'Instances', []) resource = resource[0] - ec2_run = EC2Run() - running_instances_data = ec2_run.run() - assert running_instances_data == [ - { - 'ResourceId': resource.get('InstanceId'), - 'User': 'cloud-governance', - 'SkipPolicy': 'NA', - 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), - 'InstanceType': resource.get('InstanceType'), - 'InstanceState': 'stopped', - 'StateTransitionReason': resource.get('StateTransitionReason'), - 'RunningDays': 0, - 'CleanUpDays': 4, - 'DryRun': 'no', - 'Name': 'Unittest', - 'RegionName': 'ap-south-1', - 'ResourceStopped': 'True' - } - ] + instance_run = InstanceRun() + running_instances_data = instance_run.run() + assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') + assert running_instances_data[0]['DryRun'] == 'no' + assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['CleanUpDays'] == 4 assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 - ec2_run.run() + instance_run.run() instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations'] instances = instances[0]['Instances'][0] - aws_cleanup_operations = AWSCleanUpOperations() + aws_cleanup_operations = AWSPolicyOperations() assert aws_cleanup_operations.get_tag_name_from_tags(tags=instances.get('Tags'), tag_name='DaysCount').split('@')[-1] == '0' ec2_client.start_instances(InstanceIds=[resource.get('InstanceId')]) - ec2_run.run() + instance_run.run() instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations'] instances = instances[0]['Instances'][0] assert aws_cleanup_operations.get_tag_name_from_tags(tags=instances.get('Tags'), tag_name='DaysCount').split('@')[-1] == '1' @@ -277,12 +209,12 @@ def test_ec2_run_stop_start(): @mock_iam def test_ec2_force_delete(): """ - This method tests ec2_run + This method tests instance_run :return: :rtype: """ environment_variables.environment_variables_dict['FORCE_DELETE'] = True - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'no' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -296,26 +228,12 @@ def test_ec2_force_delete(): []) resource = resource[0] environment_variables.environment_variables_dict['RESOURCE_ID'] = resource.get('InstanceId') - ec2_run = EC2Run() - running_instances_data = ec2_run.run() - assert running_instances_data == [ - { - 'ResourceId': resource.get('InstanceId'), - 'User': 'cloud-governance', - 'SkipPolicy': 'NA', - 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), - 'InstanceType': resource.get('InstanceType'), - 'InstanceState': 'stopped', - 'StateTransitionReason': resource.get('StateTransitionReason'), - 'RunningDays': 0, - 'CleanUpDays': 4, - 'DryRun': 'no', - 'Name': 'Unittest', - 'RegionName': 'ap-south-1', - 'ResourceStopped': 'True', - 'ForceDeleted': 'True' - } - ] + instance_run = InstanceRun() + running_instances_data = instance_run.run() + assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') + assert running_instances_data[0]['DryRun'] == 'no' + assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['ForceDeleted'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -324,12 +242,12 @@ def test_ec2_force_delete(): @mock_iam def test_ec2_force_delete_skip(): """ - This method tests ec2_run + This method tests instance_run :return: :rtype: """ environment_variables.environment_variables_dict['FORCE_DELETE'] = True - environment_variables.environment_variables_dict['policy'] = 'ec2_run' + environment_variables.environment_variables_dict['policy'] = 'instance_run' environment_variables.environment_variables_dict['dry_run'] = 'yes' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = 'ap-south-1' ec2_client = boto3.client('ec2', region_name='ap-south-1') @@ -343,23 +261,10 @@ def test_ec2_force_delete_skip(): []) resource = resource[0] environment_variables.environment_variables_dict['RESOURCE_ID'] = resource.get('InstanceId') - ec2_run = EC2Run() - running_instances_data = ec2_run.run() - assert running_instances_data == [ - { - 'ResourceId': resource.get('InstanceId'), - 'User': 'cloud-governance', - 'SkipPolicy': 'NA', - 'LaunchTime': resource['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), - 'InstanceType': resource.get('InstanceType'), - 'InstanceState': 'running', - 'StateTransitionReason': resource.get('StateTransitionReason'), - 'RunningDays': 0, - 'CleanUpDays': 4, - 'DryRun': 'yes', - 'Name': 'Unittest', - 'RegionName': 'ap-south-1', - 'ResourceStopped': 'False', - } - ] + instance_run = InstanceRun() + running_instances_data = instance_run.run() + assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') + assert running_instances_data[0]['DryRun'] == 'yes' + assert running_instances_data[0]['ResourceStopped'] == 'False' + assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 diff --git a/tests/unittest/cloud_governance/policy/azure/__init__.py b/tests/unittest/cloud_governance/policy/azure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/cloud_governance/policy/azure/test_instance_run.py b/tests/unittest/cloud_governance/policy/azure/test_instance_run.py new file mode 100644 index 00000000..290694ee --- /dev/null +++ b/tests/unittest/cloud_governance/policy/azure/test_instance_run.py @@ -0,0 +1,183 @@ +import datetime +from unittest.mock import patch, Mock + +from azure.mgmt.compute import ComputeManagementClient +from azure.mgmt.resource import ResourceManagementClient + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.azure.cleanup.instance_run import InstanceRun +from tests.unittest.mocks.azure.mock_compute import MockVirtualMachine, MockAzure + + +def test_instance_run(): + """ + This method tests instance_run + :return: + :rtype: + """ + vm1 = MockVirtualMachine(tags={'User': 'mock'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines = Mock() + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines): + instance_run = InstanceRun() + response = instance_run.run() + assert len(response) == 1 + response = response[0] + assert 'DryRun' in response.keys() + assert 'False' == response['ResourceStopped'] + + +def test_instance_run_stop_false(): + """ + This method tests instance_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'instance_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines): + instance_run = InstanceRun() + response = instance_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 1 == response[0]['CleanUpDays'] + assert 'False' == response[0]['ResourceStopped'] + + +def test_instance_run_stopped(): + """ + This method tests instance_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 0 + environment_variables.environment_variables_dict['policy'] = 'instance_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.begin_deallocate.side_effect = Mock() + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines): + instance_run = InstanceRun() + response = instance_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 1 == response[0]['CleanUpDays'] + assert 'True' == response[0]['ResourceStopped'] + + +def test_instance_run_stopped_skip(): + """ + This method tests instance_run skip + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 0 + environment_variables.environment_variables_dict['policy'] = 'instance_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock', 'Policy': 'notdelete'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines): + instance_run = InstanceRun() + response = instance_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() + assert 1 == response[0]['CleanUpDays'] + assert 'False' == response[0]['ResourceStopped'] + + +def test_instance_run_stopped_test_days(): + """ + This method tests instance_run skip + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'instance_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + date = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).date() + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock', 'Policy': 'notdelete', + 'DaysCount': f'{date}@1'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + mock_tags = Mock() + mock_tags.begin_create_or_update_at_scope.side_effect = mock_tags + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines), \ + patch.object(ResourceManagementClient, 'tags', mock_tags): + instance_run = InstanceRun() + response = instance_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 2 == response[0]['CleanUpDays'] + assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() + assert 'False' == response[0]['ResourceStopped'] + + +def test_instance_run_stopped_test_current_day(): + """ + This method tests instance_run skip + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'instance_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + date = (datetime.datetime.utcnow()).date() + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock', 'Policy': 'notdelete', + 'DaysCount': f'{date}@1'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + mock_tags = Mock() + mock_tags.begin_create_or_update_at_scope.side_effect = mock_tags + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines), \ + patch.object(ResourceManagementClient, 'tags', mock_tags): + instance_run = InstanceRun() + response = instance_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 1 == response[0]['CleanUpDays'] + assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() + assert 'False' == response[0]['ResourceStopped'] + + +def test_instance_run_vm_already_stopped(): + """ + This method tests instance_run already stopped + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'instance_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + date = (datetime.datetime.utcnow()).date() + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock', 'Policy': 'notdelete', + 'DaysCount': f'{date}@1'}) + mock_azure = MockAzure(vms=[vm1], status2="Vm Stopped") + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + mock_tags = Mock() + mock_tags.begin_create_or_update_at_scope.side_effect = mock_tags + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines), \ + patch.object(ResourceManagementClient, 'tags', mock_tags): + instance_run = InstanceRun() + response = instance_run.run() + assert len(response) == 0 diff --git a/tests/unittest/mocks/__init__.py b/tests/unittest/mocks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/mocks/azure/__init__.py b/tests/unittest/mocks/azure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/mocks/azure/mock_compute.py b/tests/unittest/mocks/azure/mock_compute.py new file mode 100644 index 00000000..c83b0afa --- /dev/null +++ b/tests/unittest/mocks/azure/mock_compute.py @@ -0,0 +1,48 @@ +import uuid +from datetime import datetime + +from azure.core.paging import ItemPaged +from azure.mgmt.compute.v2023_03_01.models import VirtualMachine, HardwareProfile, VirtualMachineInstanceView, \ + InstanceViewStatus + + +class MockVirtualMachine(VirtualMachine): + + def __init__(self, tags: dict = None): + super().__init__(location='mock') + self.tags = tags if tags else {} + self.name = 'mock_machine' + self.time_created = datetime.utcnow() + self.hardware_profile = HardwareProfile(vm_size='Standard_D2s_v3') + self.id = f'/subscriptions/{uuid.uuid1()}/resourceGroups/mock/providers/Microsoft.Compute/virtualMachines/mock-machine' + + +class MockVirtualMachineInstanceView(VirtualMachineInstanceView): + + def __init__(self, status1: str = "Unknown", status2: str = 'Vm Running'): + super().__init__() + self.statuses = [ + InstanceViewStatus(display_status=status1), + InstanceViewStatus(display_status=status2) + ] + + +class CustomItemPaged(ItemPaged): + + def __init__(self, vms_list: list = None): + super().__init__() + self._page_iterator = iter(vms_list if vms_list else []) + + +class MockAzure: + + def __init__(self, vms: list = None, status1: str = "Unknown", status2: str = 'Vm Running'): + self.vms = vms if vms else [] + self.status1 = status1 + self.status2 = status2 + + def mock_list_all(self, *args, **kwargs): + return CustomItemPaged(vms_list=self.vms) + + def mock_instance_view(self, *args, **kwargs): + return MockVirtualMachineInstanceView(status1=self.status1, status2=self.status2)