From a7354d45118f1455668bba4555eef57b29156b52 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Thu, 21 Dec 2023 16:38:51 +0530 Subject: [PATCH] Added the azure ec2_run policy --- .../azure/compute/compute_operations.py | 37 +++- .../helpers/aws/aws_cleanup_operations.py | 29 +-- .../common/helpers/azure/__init__.py | 0 .../helpers/azure/azure_cleanup_operations.py | 83 ++++++++ .../common/helpers/cleanup_operations.py | 44 ++++- cloud_governance/common/utils/utils.py | 50 +++++ cloud_governance/main/aws_main_operations.py | 41 ---- cloud_governance/main/main.py | 9 +- .../main/main_oerations/__init__.py | 0 .../main/main_oerations/main_operations.py | 44 +++++ .../policy/aws/cleanup/ec2_run.py | 29 +-- .../policy/azure/cleanup/__init__.py | 0 .../policy/azure/cleanup/vm_run.py | 77 ++++++++ .../policy_runners/aws/policy_runner.py | 55 +++--- .../policy/policy_runners/aws/upload_s3.py | 1 + .../policy/policy_runners/azure/__init__.py | 0 .../policy_runners/azure/policy_runner.py | 30 +++ .../common/abstract_policy_runner.py | 63 ++++-- .../elasticsearch/upload_elastic_search.py | 6 +- .../clouds/aws/s3/test_s3_operations.py | 17 +- .../helpers/aws/test_aws_cleaup_operations.py | 6 +- .../policy/aws/cleanup/test_ec2_run.py | 21 +- .../cloud_governance/policy/azure/__init__.py | 0 .../policy/azure/test_vm_run.py | 183 ++++++++++++++++++ tests/unittest/mocks/__init__.py | 0 tests/unittest/mocks/azure/__init__.py | 0 tests/unittest/mocks/azure/mock_compute.py | 48 +++++ 27 files changed, 721 insertions(+), 152 deletions(-) create mode 100644 cloud_governance/common/helpers/azure/__init__.py create mode 100644 cloud_governance/common/helpers/azure/azure_cleanup_operations.py create mode 100644 cloud_governance/common/utils/utils.py delete mode 100644 cloud_governance/main/aws_main_operations.py create mode 100644 cloud_governance/main/main_oerations/__init__.py create mode 100644 cloud_governance/main/main_oerations/main_operations.py create mode 100644 cloud_governance/policy/azure/cleanup/__init__.py create mode 100644 cloud_governance/policy/azure/cleanup/vm_run.py create mode 100644 cloud_governance/policy/policy_runners/azure/__init__.py create mode 100644 cloud_governance/policy/policy_runners/azure/policy_runner.py create mode 100644 tests/unittest/cloud_governance/policy/azure/__init__.py create mode 100644 tests/unittest/cloud_governance/policy/azure/test_vm_run.py create mode 100644 tests/unittest/mocks/__init__.py create mode 100644 tests/unittest/mocks/azure/__init__.py create mode 100644 tests/unittest/mocks/azure/mock_compute.py diff --git a/cloud_governance/common/clouds/azure/compute/compute_operations.py b/cloud_governance/common/clouds/azure/compute/compute_operations.py index 3ca52a266..dee906a08 100644 --- a/cloud_governance/common/clouds/azure/compute/compute_operations.py +++ b/cloud_governance/common/clouds/azure/compute/compute_operations.py @@ -31,9 +31,9 @@ def get_all_instances(self) -> [VirtualMachine]: instances_list: [VirtualMachine] = self._item_paged_iterator(item_paged_object=instances_paged_object) return instances_list - def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine: + def get_instance_statuses(self, resource_id: str, vm_name: str) -> dict: """ - This method returns the virtual machine data by taking the id + This method returns the virtual machine instance status :param vm_name: :type vm_name: :param resource_id: @@ -42,6 +42,33 @@ def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine: :rtype: """ resource_group_name = self._get_resource_group_name_from_resource_id(resource_id=resource_id) - virtual_machine = self.__compute_client.virtual_machines.get(resource_group_name=resource_group_name, - vm_name=vm_name) - return virtual_machine + virtual_machine = self.__compute_client.virtual_machines.instance_view(resource_group_name=resource_group_name, + vm_name=vm_name) + return virtual_machine.as_dict() + + def get_id_dict_data(self, resource_id: str): + """ + This method generates the vm id dictionary + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + pairs = resource_id.split('/')[1:] + key_pairs = {pairs[i].lower(): pairs[i + 1] for i in range(0, len(pairs), 2)} + return key_pairs + + def stop_vm(self, resource_id: str): + """ + This method stops the vm + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + id_key_pairs = self.get_id_dict_data(resource_id) + resource_group_name = id_key_pairs.get('resourcegroups') + vm_name = id_key_pairs.get('virtualmachines') + status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name, + vm_name=vm_name) + return status.done() diff --git a/cloud_governance/common/helpers/aws/aws_cleanup_operations.py b/cloud_governance/common/helpers/aws/aws_cleanup_operations.py index 4539ade50..313936501 100644 --- a/cloud_governance/common/helpers/aws/aws_cleanup_operations.py +++ b/cloud_governance/common/helpers/aws/aws_cleanup_operations.py @@ -1,8 +1,9 @@ -import datetime import boto3 +from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations +from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations from cloud_governance.common.logger.init_logger import logger @@ -12,8 +13,11 @@ class AWSCleanUpOperations(AbstractCleanUpOperations): def __init__(self): super().__init__() self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') + self._cloud_name = 'AWS' self.__s3operations = S3Operations(region_name=self._region) self._ec2_client = boto3.client('ec2', region_name=self._region) + self._ec2_operations = EC2Operations(region=self._region) + self._es_upload = ElasticUpload() self._s3_client = boto3.client('s3') self._iam_client = boto3.client('iam') @@ -33,23 +37,6 @@ def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str: return tag.get('Value').strip() return '' - def get_clean_up_days_count(self, tags: list): - """ - This method returns the cleanup days count - :param tags: - :type tags: - :return: - :rtype: - """ - last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount') - if not last_used_day: - return 1 - else: - date, days = last_used_day.split('@') - if date != str(self.CURRENT_DATE): - return int(days) + 1 - return 1 if int(days) == 0 else int(days) - def _delete_resource(self, resource_id: str): """ This method deletes the resource by verifying the policy @@ -93,9 +80,9 @@ def __remove_tag_key_aws(self, tags: list): custom_tags.append(tag) return custom_tags - def __update_tag_value(self, tags: list, tag_name: str, tag_value: str): + def _update_tag_value(self, tags: list, tag_name: str, tag_value: str): """ - This method updates the tag_value + This method returns the updated tag_list by adding the tag_name and tag_value to the tags @param tags: @param tag_name: @param tag_value: @@ -133,7 +120,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag :return: :rtype: """ - tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) + tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) try: if self._policy == 's3_inactive': self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags}) diff --git a/cloud_governance/common/helpers/azure/__init__.py b/cloud_governance/common/helpers/azure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/common/helpers/azure/azure_cleanup_operations.py b/cloud_governance/common/helpers/azure/azure_cleanup_operations.py new file mode 100644 index 000000000..6e9199a84 --- /dev/null +++ b/cloud_governance/common/helpers/azure/azure_cleanup_operations.py @@ -0,0 +1,83 @@ + +from cloud_governance.common.clouds.azure.compute.compute_operations import ComputeOperations +from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations +from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.common.utils.utils import Utils + + +class AzureCleaUpOperations(AbstractCleanUpOperations): + + def __init__(self): + super().__init__() + self._cloud_name = 'Azure' + self.compute_operations = ComputeOperations() + self.resource_group_operations = ResourceGroupOperations() + + def get_tag_name_from_tags(self, tags: dict, tag_name: str): + """ + This method returns the tag value by the tag_name + :param tags: + :type tags: + :param tag_name: + :type tag_name: + :return: + :rtype: + """ + if tags: + for key, value in tags.items(): + if Utils.equal_ignore_case(key, tag_name): + return value + return '' + + def _delete_resource(self, resource_id: str): + """ + This method deletes the + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + action = "deleted" + try: + if self._policy == 'vm_run': + action = "Stopped" + self.compute_operations.stop_vm(resource_id=resource_id) + logger.info(f'{self._policy} {action}: {resource_id}') + except Exception as err: + logger.info(f'Exception raised: {err}: {resource_id}') + + def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict): + tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) + try: + if self._policy == 'vm_run': + self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags) + except Exception as err: + logger.info(f'Exception raised: {err}: {resource_id}') + + def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str): + """ + This method returns the updated tag_list by adding the tag_name and tag_value to the tags + @param tags: + @param tag_name: + @param tag_value: + @return: + """ + if self._dry_run == "yes": + tag_value = 0 + tag_value = f'{self.CURRENT_DATE}@{tag_value}' + found = False + updated_tags = {} + if tags: + for key, value in tags.items(): + if Utils.equal_ignore_case(key, tag_name): + if value.split("@")[0] != self.CURRENT_DATE: + updated_tags[key] = tag_value + else: + if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1: + updated_tags[key] = tag_value + found = True + tags.update(updated_tags) + if not found: + return {tag_name: tag_value} + return tags diff --git a/cloud_governance/common/helpers/cleanup_operations.py b/cloud_governance/common/helpers/cleanup_operations.py index aa8ac1521..2622fc724 100644 --- a/cloud_governance/common/helpers/cleanup_operations.py +++ b/cloud_governance/common/helpers/cleanup_operations.py @@ -14,13 +14,27 @@ class AbstractCleanUpOperations(ABC): def __init__(self): self._environment_variables_dict = environment_variables.environment_variables_dict + self.account = self._environment_variables_dict.get('account') self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION') self._dry_run = self._environment_variables_dict.get('dry_run') self._policy = self._environment_variables_dict.get('policy') self._force_delete = self._environment_variables_dict.get('FORCE_DELETE') self._resource_id = self._environment_variables_dict.get('RESOURCE_ID') - @abstractmethod + def calculate_days(self, create_date: Union[datetime, str]): + """ + This method returns the days + :param create_date: + :type create_date: + :return: + :rtype: + """ + if isinstance(create_date, str): + create_date = datetime.strptime(create_date, "%Y-%M-%d") + today = datetime.utcnow().date() + days = today - create_date.date() + return days.days + def get_clean_up_days_count(self, tags: Union[list, dict]): """ This method returns the cleanup days count @@ -29,7 +43,16 @@ def get_clean_up_days_count(self, tags: Union[list, dict]): :return: :rtype: """ - raise NotImplementedError("This method is Not yet implemented") + if self._dry_run == 'yes': + return 0 + last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount') + if not last_used_day: + return 1 + else: + date, days = last_used_day.split('@') + if date != str(self.CURRENT_DATE): + return int(days) + 1 + return 1 if int(days) == 0 else int(days) @abstractmethod def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str): @@ -85,7 +108,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag """ raise NotImplementedError("This method is Not yet implemented") - def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int, + def verify_and_delete_resource(self, resource_id: str, tags: Union[list, dict], clean_up_days: int, days_to_delete_resource: int = None, **kwargs): """ This method verify and delete the resource by calculating the days @@ -109,3 +132,18 @@ def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days self._delete_resource(resource_id=resource_id) cleanup_resources = True return cleanup_resources + + @abstractmethod + def _update_tag_value(self, tags: Union[list, dict], tag_name: str, tag_value: str): + """ + This method returns the updated tag_list by adding the tag_name and tag_value to the tags + :param tags: + :type tags: + :param tag_name: + :type tag_name: + :param tag_value: + :type tag_value: + :return: + :rtype: + """ + raise NotImplementedError("This method is Not yet implemented") diff --git a/cloud_governance/common/utils/utils.py b/cloud_governance/common/utils/utils.py new file mode 100644 index 000000000..49e1ea6ee --- /dev/null +++ b/cloud_governance/common/utils/utils.py @@ -0,0 +1,50 @@ + +import os + + +class Utils: + + def __init__(self): + pass + + @staticmethod + def get_cloud_policies(cloud_name: str, file_type: str = '.py', dir_dict: bool = False, + exclude_policies: list = None): + """ + This method returns the policies by cloud_name + :return: + :rtype: + """ + exclude_policies = [] if not exclude_policies else exclude_policies + policies_dict = {} + policies_list = [] + policies_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'policy', cloud_name) + for (dir_path, _, filenames) in os.walk(policies_path): + immediate_parent = dir_path.split("/")[-1] + for filename in filenames: + if not filename.startswith('__') and filename.endswith(file_type): + filename = os.path.splitext(filename)[0] + if filename not in exclude_policies: + if dir_dict: + policies_dict.setdefault(immediate_parent, []).append(filename) + else: + policies_list.append(filename) + return policies_dict if dir_dict else policies_list + + @staticmethod + def equal_ignore_case(str1: str, str2: str, *args): + """ + This method returns boolean by comparing equal in-case sensitive all strings + :param str1: + :type str1: + :param str2: + :type str2: + :param args: + :type args: + :return: + :rtype: + """ + equal = str1.lower() == str2.lower() + for val in args: + equal = str1.lower() == val.lower() and equal + return equal diff --git a/cloud_governance/main/aws_main_operations.py b/cloud_governance/main/aws_main_operations.py deleted file mode 100644 index 60823ff31..000000000 --- a/cloud_governance/main/aws_main_operations.py +++ /dev/null @@ -1,41 +0,0 @@ -import os - -from cloud_governance.main.environment_variables import environment_variables -from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner - - -class AWSMainOperations: - - def __init__(self): - self.__environment_variables_dict = environment_variables.environment_variables_dict - self.__policy = self.__environment_variables_dict.get('policy', '') - self.__policy_runner = PolicyRunner() - - def __get_policies(self) -> dict: - """ - This method gets the aws policies - :return: - :rtype: - """ - policies = {} - policies_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'policy', 'aws') - for (dirpath, dirnames, filenames) in os.walk(policies_path): - immediate_parent = dirpath.split("/")[-1] - for filename in filenames: - if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')): - policies.setdefault(immediate_parent, []).append(os.path.splitext(filename)[0]) - return policies - - def run(self): - """ - This method run the AWS Policy operations - :return: - :rtype: - """ - policies_list = self.__get_policies() - for policy_type, policies in policies_list.items(): - # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement - if self.__policy in policies and self.__policy == "ec2_run": - self.__policy_runner.run(source=policy_type) - return True - return False diff --git a/cloud_governance/main/main.py b/cloud_governance/main/main.py index 2f40633ea..8b70852ee 100644 --- a/cloud_governance/main/main.py +++ b/cloud_governance/main/main.py @@ -3,9 +3,8 @@ from ast import literal_eval # str to dict import boto3 # regions -from cloud_governance.cloud_resource_orchestration.monitor.cloud_monitor import CloudMonitor -from cloud_governance.main.aws_main_operations import AWSMainOperations from cloud_governance.main.main_common_operations import run_common_policies +from cloud_governance.main.main_oerations.main_operations import MainOperations from cloud_governance.main.run_cloud_resource_orchestration import run_cloud_resource_orchestration from cloud_governance.policy.policy_operations.aws.cost_expenditure.cost_report_policies import CostReportPolicies from cloud_governance.policy.policy_operations.azure.azure_policy_runner import AzurePolicyRunner @@ -206,10 +205,8 @@ def main(): es_index = environment_variables_dict.get('es_index', '') es_doc_type = environment_variables_dict.get('es_doc_type', '') bucket = environment_variables_dict.get('bucket', '') - response = False - if is_policy_aws(): - aws_main_operations = AWSMainOperations() - response = aws_main_operations.run() + main_operations = MainOperations() + response = main_operations.run() if not response: if environment_variables_dict.get('COMMON_POLICIES'): run_common_policies() diff --git a/cloud_governance/main/main_oerations/__init__.py b/cloud_governance/main/main_oerations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/main/main_oerations/main_operations.py b/cloud_governance/main/main_oerations/main_operations.py new file mode 100644 index 000000000..b6fc2e203 --- /dev/null +++ b/cloud_governance/main/main_oerations/main_operations.py @@ -0,0 +1,44 @@ + +from cloud_governance.common.utils.utils import Utils +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.policy_runners.azure.policy_runner import PolicyRunner as AzurePolicyRunner +from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner as AWSPolicyRunner + + +class MainOperations: + + def __init__(self): + self.utils = Utils() + self._environment_variables_dict = environment_variables.environment_variables_dict + self._policy = self._environment_variables_dict.get('policy', '') + self._public_cloud_name = self._environment_variables_dict.get('PUBLIC_CLOUD_NAME', '') + + def get_policy_runner(self): + """ + This method returns the cloud policy runner object + :return: + :rtype: + """ + policy_runner = None + if Utils.equal_ignore_case(self._public_cloud_name, 'AWS'): + policy_runner = AWSPolicyRunner() + else: + if Utils.equal_ignore_case(self._public_cloud_name, 'AZURE'): + policy_runner = AzurePolicyRunner() + + return policy_runner + + def run(self): + """ + This method run the AWS Policy operations + :return: + :rtype: + """ + policies_list = Utils.get_cloud_policies(cloud_name=self._public_cloud_name, dir_dict=True) + policy_runner = self.get_policy_runner() + for policy_type, policies in policies_list.items(): + # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement + if self._policy in policies and self._policy in ("vm_run", "ec2_run"): + policy_runner.run(source=policy_type) + return True + return False diff --git a/cloud_governance/policy/aws/cleanup/ec2_run.py b/cloud_governance/policy/aws/cleanup/ec2_run.py index 4b6ee19f3..4bc355e9b 100644 --- a/cloud_governance/policy/aws/cleanup/ec2_run.py +++ b/cloud_governance/policy/aws/cleanup/ec2_run.py @@ -1,10 +1,10 @@ import datetime -from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import \ - NonClusterZombiePolicy +from cloud_governance.common.helpers.aws.aws_cleanup_operations import AWSCleanUpOperations -class EC2Run(NonClusterZombiePolicy): + +class EC2Run(AWSCleanUpOperations): RESOURCE_ACTION = "Stopped" @@ -31,8 +31,8 @@ def __update_instance_type_count(self, instances: list): 'instance_count': value, 'timestamp': datetime.datetime.utcnow(), 'region': self._region, - 'account': self._account.upper().replace('OPENSHIFT-', ''), - 'index_id': f'{key}-{self._account.lower()}-{self._region}-{str(datetime.datetime.utcnow().date())}' + 'account': self.account.upper().replace('OPENSHIFT-', ''), + 'index_id': f'{key}-{self.account.lower()}-{self._region}-{str(datetime.datetime.utcnow().date())}' }) self._es_upload.es_upload_data(items=es_instance_types_data, es_index=self.__es_index, set_index='index_id') @@ -48,15 +48,15 @@ def __ec2_run(self): for instance in instances: tags = instance.get('Tags', []) if instance.get('State', {}).get('Name') == 'running': - running_days = self._calculate_days(instance.get('LaunchTime')) - cleanup_days = self._aws_cleanup_policies.get_clean_up_days_count(tags=tags) - cleanup_result = self._aws_cleanup_policies.verify_and_delete_resource( + running_days = self.calculate_days(instance.get('LaunchTime')) + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource( resource_id=instance.get('InstanceId'), tags=tags, clean_up_days=cleanup_days) resource_data = { 'ResourceId': instance.get('InstanceId'), - 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'), - 'SkipPolicy': self._aws_cleanup_policies.get_skip_policy_value(tags=tags), + 'User': self.get_tag_name_from_tags(tags=tags, tag_name='User'), + 'SkipPolicy': self.get_skip_policy_value(tags=tags), 'LaunchTime': instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"), 'InstanceType': instance.get('InstanceType'), 'InstanceState': instance.get('State', {}).get('Name') if not cleanup_result else 'stopped', @@ -64,17 +64,18 @@ def __ec2_run(self): 'RunningDays': running_days, 'CleanUpDays': cleanup_days, 'DryRun': self._dry_run, - 'Name': self._get_tag_name_from_tags(tags=tags, tag_name='Name'), + 'Name': self.get_tag_name_from_tags(tags=tags, tag_name='Name'), 'RegionName': self._region, - f'Resource{self.RESOURCE_ACTION}': str(cleanup_result) + f'Resource{self.RESOURCE_ACTION}': str(cleanup_result), + 'PublicCloud': self._cloud_name } if self._force_delete and self._dry_run == 'no': resource_data.update({'ForceDeleted': str(self._force_delete)}) running_instances_data.append(resource_data) else: cleanup_days = 0 - self._aws_cleanup_policies.update_resource_day_count_tag(resource_id=instance.get('InstanceId'), - cleanup_days=cleanup_days, tags=tags) + self.update_resource_day_count_tag(resource_id=instance.get('InstanceId'), cleanup_days=cleanup_days, + tags=tags) return running_instances_data diff --git a/cloud_governance/policy/azure/cleanup/__init__.py b/cloud_governance/policy/azure/cleanup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/policy/azure/cleanup/vm_run.py b/cloud_governance/policy/azure/cleanup/vm_run.py new file mode 100644 index 000000000..99401fcaa --- /dev/null +++ b/cloud_governance/policy/azure/cleanup/vm_run.py @@ -0,0 +1,77 @@ +from cloud_governance.common.helpers.azure.azure_cleanup_operations import AzureCleaUpOperations + + +class VmRun(AzureCleaUpOperations): + + RESOURCE_ACTION = "Stopped" + + def __init__(self): + super().__init__() + + def __get_instance_status(self, resource_id: str, vm_name: str): + """ + This method returns the VM status of the Virtual Machine + :param resource_id: + :type resource_id: + :param vm_name: + :type vm_name: + :return: + :rtype: + """ + instance_statuses = self.compute_operations.get_instance_statuses(resource_id=resource_id, vm_name=vm_name) + statuses = instance_statuses.get('statuses', {}) + if len(statuses) >= 2: + status = statuses[1].get('display_status', '').lower() + elif len(statuses) == 1: + status = statuses[0].get('display_status', '').lower() + else: + status = 'Unknown Status' + return status + + def __vm_run(self): + """ + This method returns the running vms in the AAzure cloud and stops based on the action + :return: + :rtype: + """ + vms_list = self.compute_operations.get_all_instances() + running_vms = [] + for vm in vms_list: + status = self.__get_instance_status(resource_id=vm.id, vm_name=vm.name) + tags = vm.tags if vm.tags else {} + if 'running' in status: + running_days = self.calculate_days(vm.time_created) + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource(resource_id=vm.id, tags=tags, + clean_up_days=cleanup_days) + resource_data = { + 'ResourceId': vm.name, + 'VmId': vm.vm_id, + 'User': self.get_tag_name_from_tags(tags=tags, tag_name='User'), + 'SkipPolicy': self.get_skip_policy_value(tags=tags), + 'LaunchTime': vm.time_created, + 'InstanceType': vm.hardware_profile.vm_size, + 'InstanceState': status if cleanup_result else 'Vm Stopped', + 'RunningDays': running_days, + 'CleanUpDays': cleanup_days, + 'DryRun': self._dry_run, + 'Name': vm.name, + 'RegionName': vm.location, + f'Resource{self.RESOURCE_ACTION}': str(cleanup_result), + 'PublicCloud': self._cloud_name + } + if self._force_delete and self._dry_run == 'no': + resource_data.update({'ForceDeleted': str(self._force_delete)}) + running_vms.append(resource_data) + else: + cleanup_days = 0 + self.update_resource_day_count_tag(resource_id=vm.id, cleanup_days=cleanup_days, tags=tags) + return running_vms + + def run(self): + """ + This method starts the VMRun operations + :return: + :rtype: + """ + return self.__vm_run() diff --git a/cloud_governance/policy/policy_runners/aws/policy_runner.py b/cloud_governance/policy/policy_runners/aws/policy_runner.py index f09f77cbd..3e2076245 100644 --- a/cloud_governance/policy/policy_runners/aws/policy_runner.py +++ b/cloud_governance/policy/policy_runners/aws/policy_runner.py @@ -2,6 +2,8 @@ import importlib import inspect +from typing import Callable + from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations from cloud_governance.common.logger.init_logger import logger from cloud_governance.policy.policy_runners.aws.upload_s3 import UploadS3 @@ -13,40 +15,37 @@ class PolicyRunner(AbstractPolicyRunner): def __init__(self): super().__init__() - def run(self, source: str = "", upload: bool = True): + def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: bool): """ - This method run the AWS policies classes + This method executes the policy + :param policy_class_name: + :type policy_class_name: + :param run_policy: + :type run_policy: :param upload: :type upload: - :param source: - :type source: :return: :rtype: """ - source_policy = f"{source}.{self._policy}" if source else self._policy - logger.info(f'account={self._account}, policy={self._policy}, dry_run={self._dry_run}') - zombie_non_cluster_policy_module = importlib.import_module(f'cloud_governance.policy.aws.{source_policy}') - policy_result = [] ec2_operations = EC2Operations() upload_to_s3 = UploadS3() - for cls in inspect.getmembers(zombie_non_cluster_policy_module, inspect.isclass): - if self._policy.replace('_', '').replace('-', '') == cls[0].lower(): - active_regions = [self._region] - if self._run_active_regions: - active_regions = ec2_operations.get_active_regions() - logger.info("Running the policy in All AWS active regions") - for active_region in active_regions: - logger.info(f"Running the {self._policy} in Region: {active_region}") - self._environment_variables_dict['AWS_DEFAULT_REGION'] = active_region - response = cls[1]().run() - if isinstance(response, str): - logger.info(f'key: {cls[0]}, Response: {response}') - else: - policy_result.extend(response) - logger.info(f'key: {cls[0]}, count: {len(response)}, {response}') - if upload: - self._upload_elastic_search.upload(data=response) - upload_to_s3.upload(data=response) - if self._save_to_file_path: - self.write_to_file(data=policy_result) + active_regions = [self._region] + if self._run_active_regions: + active_regions = ec2_operations.get_active_regions() + logger.info("Running the policy in All AWS active regions") + for active_region in active_regions: + logger.info(f"Running the {self._policy} in Region: {active_region}") + self._environment_variables_dict['AWS_DEFAULT_REGION'] = active_region + response = run_policy().run() + print(response) + if isinstance(response, str): + logger.info(f'key: {policy_class_name}, Response: {response}') + else: + policy_result.extend(response) + logger.info(f'key: {policy_class_name}, count: {len(response)}, {response}') + if upload: + self._upload_elastic_search.upload(data=response) + upload_to_s3.upload(data=response) + return policy_result + diff --git a/cloud_governance/policy/policy_runners/aws/upload_s3.py b/cloud_governance/policy/policy_runners/aws/upload_s3.py index 6abd3ddc1..c8a551f89 100644 --- a/cloud_governance/policy/policy_runners/aws/upload_s3.py +++ b/cloud_governance/policy/policy_runners/aws/upload_s3.py @@ -23,6 +23,7 @@ def upload(self, data: Union[list, dict]): """ if self._policy_output: data = json.dumps(data, cls=JsonDateTimeEncoder) + print(data) self._s3operations.save_results_to_s3(policy=self._policy.replace('_', '-'), policy_output=self._policy_output, policy_result=data) logger.info(f"Uploaded the data s3 Bucket: {self._policy_output}") diff --git a/cloud_governance/policy/policy_runners/azure/__init__.py b/cloud_governance/policy/policy_runners/azure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/policy/policy_runners/azure/policy_runner.py b/cloud_governance/policy/policy_runners/azure/policy_runner.py new file mode 100644 index 000000000..3775202b4 --- /dev/null +++ b/cloud_governance/policy/policy_runners/azure/policy_runner.py @@ -0,0 +1,30 @@ +from typing import Callable + +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.policy.policy_runners.common.abstract_policy_runner import AbstractPolicyRunner + + +class PolicyRunner(AbstractPolicyRunner): + + def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: bool = False): + """ + This method executes the policy + :param policy_class_name: + :type policy_class_name: + :param run_policy: + :type run_policy: + :param upload: + :type upload: + :return: + :rtype: + """ + policy_result = [] + response = run_policy().run() + if isinstance(response, str): + logger.info(response) + else: + policy_result.extend(response) + return policy_result + + def __init__(self): + super().__init__() diff --git a/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py b/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py index 63464c267..a4a7d3b4e 100644 --- a/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py +++ b/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py @@ -1,6 +1,8 @@ +import importlib +import inspect import os.path from abc import abstractmethod, ABC -from typing import Union +from typing import Union, Callable from cloud_governance.common.logger.init_logger import logger from cloud_governance.main.environment_variables import environment_variables @@ -18,12 +20,9 @@ def __init__(self): self._run_active_regions = self._environment_variables_dict.get('RUN_ACTIVE_REGIONS') self._upload_elastic_search = UploadElasticSearch() self._save_to_file_path = self._environment_variables_dict.get('SAVE_TO_FILE_PATH') + self._public_cloud_name = self._environment_variables_dict.get('PUBLIC_CLOUD_NAME', '') - @abstractmethod - def run(self): - raise NotImplementedError("This method is not yet implemented") - - def write_to_file(self, data: Union[list, dict]): + def write_to_file(self, data: Union[list, dict, str]): """ This method writes the data to file_path passed by the env SAVE_TO_FILE_PATH :param data: @@ -39,20 +38,58 @@ def write_to_file(self, data: Union[list, dict]): with open(file_name, 'w') as file: if isinstance(data, list): for item in data: - if not header_added: - keys = [str(val) for val in list(item.keys())] + ["\n"] - file.write(', '.join(keys)) - header_added = True - values = [str(val) for val in list(item.values())] + ["\n"] - file.write(', '.join(values)) + if isinstance(item, dict): + if not header_added: + keys = [str(val) for val in list(item.keys())] + ["\n"] + file.write(', '.join(keys)) + header_added = True + values = [str(val) for val in list(item.values())] + ["\n"] + file.write(', '.join(values)) + else: + file.write(f'{item}\n') else: if isinstance(data, dict): if not header_added: keys = [str(val) for val in list(data.keys())] + ["\n"] file.write(', '.join(keys)) - header_added = True values = [str(val) for val in list(data.values())] + ["\n"] file.write(', '.join(values)) + else: + file.write(data) + file.write('\n') logger.info(f"Written the data into the file_name: {file_name}") else: raise FileExistsError(f"FilePath not exists {self._save_to_file_path}") + + @abstractmethod + def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: bool): + """ + This method execute the policy + :return: + :rtype: + """ + raise NotImplementedError("This method is not yet implemented") + + def run(self, source: str = "", upload: bool = True): + """ + This method starts the method operations + :param source: + :type source: + :param upload: + :type upload: + :return: + :rtype: + """ + source_policy = f"{source}.{self._policy}" if source else self._policy + logger.info(f'CloudName={self._public_cloud_name}, account={self._account}, policy={self._policy}, dry_run={self._dry_run}') + policies_path = f'cloud_governance.policy.{self._public_cloud_name.lower()}.{source_policy}' + cloud_policies = importlib.import_module(policies_path) + policy_result = [] + + for cls in inspect.getmembers(cloud_policies, inspect.isclass): + if self._policy.replace('_', '').replace('-', '') == cls[0].lower(): + response = self.execute_policy(policy_class_name=cls[0], run_policy=cls[1], upload=upload) + policy_result.extend(response) + if self._save_to_file_path: + self.write_to_file(data=policy_result) + diff --git a/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py b/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py index a98e0ada0..dfd56e723 100644 --- a/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py +++ b/cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py @@ -30,8 +30,10 @@ def upload(self, data: Union[list, dict]): self._es_operations.upload_data_in_bulk(data_items=data.copy(), index=self._es_index) else: for policy_dict in data: - policy_dict['region_name'] = self._region - policy_dict['account'] = self._account + if 'RegionName' not in policy_dict: + policy_dict['RegionName'] = self._region + if 'account' not in policy_dict: + policy_dict['account'] = self._account self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index) logger.info(f'Uploaded the policy results to elasticsearch index: {self._es_index}') else: diff --git a/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py b/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py index ae962bc92..7639f3e2d 100644 --- a/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py +++ b/tests/unittest/cloud_governance/common/clouds/aws/s3/test_s3_operations.py @@ -9,9 +9,8 @@ # walk around for moto DeprecationWarning import warnings -from cloud_governance.main.aws_main_operations import AWSMainOperations from cloud_governance.main.environment_variables import environment_variables -from cloud_governance.policy.aws.cleanup.ec2_run import EC2Run +from cloud_governance.main.main_oerations.main_operations import MainOperations with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -195,14 +194,14 @@ def test_get_s3_latest_policy_file(): environment_variables.environment_variables_dict['dry_run'] = 'yes' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name environment_variables.environment_variables_dict['policy_output'] = f's3://{bucket_name}/tests' - ec2_client = boto3.client('ec2', region_name='ap-south-1') + ec2_client = boto3.client('ec2', region_name=region_name) default_ami_id = 'ami-03cf127a' tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) - aws_main_operations = AWSMainOperations() - aws_main_operations.run() + main_operations = MainOperations() + main_operations.run() current_date = datetime.datetime.now().date().__str__().replace('-', '/') prefix = f'tests/{region_name}/ec2-run/{current_date}' s3_operations = S3Operations(region_name=region_name, bucket=bucket_name, logs_bucket_key='tests') @@ -223,15 +222,15 @@ def test_get_last_s3_policy_content(): environment_variables.environment_variables_dict['dry_run'] = 'yes' environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name environment_variables.environment_variables_dict['policy_output'] = f's3://{bucket_name}/tests' - ec2_client = boto3.client('ec2', region_name='ap-south-1') + ec2_client = boto3.client('ec2', region_name=region_name) default_ami_id = 'ami-03cf127a' tags = [{'Key': 'User', 'Value': 'cloud-governance'}, {'Key': "Name", "Value": "Unittest"}] ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]).get('Instances', []) - aws_main_operations = AWSMainOperations() - aws_main_operations.run() + main_operations = MainOperations() + main_operations.run() current_date = datetime.datetime.now().date().__str__().replace('-', '/') key_prefix = f'tests/{region_name}/ec2-run/{current_date}' - s3_operations = S3Operations(region_name='us-east-1', bucket=bucket_name, logs_bucket_key='tests') + s3_operations = S3Operations(region_name=region_name, bucket=bucket_name, logs_bucket_key='tests') assert s3_operations.get_last_s3_policy_content(policy='ec2-run', file_name='resources.json', key_prefix=key_prefix) diff --git a/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py b/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py index e5006a908..cb908b113 100644 --- a/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py +++ b/tests/unittest/cloud_governance/common/helpers/aws/test_aws_cleaup_operations.py @@ -35,7 +35,7 @@ def test_get_clean_up_days_count(): aws_cleanup_operations = AWSCleanUpOperations() tags = [{'Key': "Name", "Value": "Unittest"}] days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) - assert days_count == 1 + assert days_count == 0 @mock_ec2 @@ -52,7 +52,7 @@ def test_get_clean_up_days_count_already_exists(): mock_date = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).date() tags = [{'Key': "Name", "Value": "Unittest"}, {'Key': "DaysCount", "Value": f'{mock_date}@1'}] days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) - assert days_count == 2 + assert days_count == 0 @mock_ec2 @@ -69,7 +69,7 @@ def test_get_clean_up_days_count_already_updated_today(): mock_date = str(datetime.datetime.utcnow().date()) tags = [{'Key': "Name", "Value": "Unittest"}, {'Key': "DaysCount", "Value": f'{mock_date}@1'}] days_count = aws_cleanup_operations.get_clean_up_days_count(tags=tags) - assert days_count == 1 + assert days_count == 0 @mock_ec2 diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py index 1f485956f..4efa3b937 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_ec2_run.py @@ -70,7 +70,8 @@ def test_ec2_run_alert(): 'DryRun': 'no', 'Name': 'Unittest', 'RegionName': 'ap-south-1', - 'ResourceStopped': 'False' + 'ResourceStopped': 'False', + 'PublicCloud': 'AWS' } ] assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 @@ -116,7 +117,8 @@ def test_ec2_run_alert_stopped(): 'DryRun': 'no', 'Name': 'Unittest', 'RegionName': 'ap-south-1', - 'ResourceStopped': 'True' + 'ResourceStopped': 'True', + 'PublicCloud': 'AWS' } ] assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -161,7 +163,8 @@ def test_ec2_run_alert_skip(): 'DryRun': 'no', 'Name': 'Unittest', 'RegionName': 'ap-south-1', - 'ResourceStopped': 'False' + 'ResourceStopped': 'False', + 'PublicCloud': 'AWS' } ] assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 @@ -206,7 +209,8 @@ def test_ec2_run_stop_reset(): 'DryRun': 'no', 'Name': 'Unittest', 'RegionName': 'ap-south-1', - 'ResourceStopped': 'True' + 'ResourceStopped': 'True', + 'PublicCloud': 'AWS' } ] assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -256,7 +260,8 @@ def test_ec2_run_stop_start(): 'DryRun': 'no', 'Name': 'Unittest', 'RegionName': 'ap-south-1', - 'ResourceStopped': 'True' + 'ResourceStopped': 'True', + 'PublicCloud': 'AWS' } ] assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -313,7 +318,8 @@ def test_ec2_force_delete(): 'Name': 'Unittest', 'RegionName': 'ap-south-1', 'ResourceStopped': 'True', - 'ForceDeleted': 'True' + 'ForceDeleted': 'True', + 'PublicCloud': 'AWS' } ] assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -355,11 +361,12 @@ def test_ec2_force_delete_skip(): 'InstanceState': 'running', 'StateTransitionReason': resource.get('StateTransitionReason'), 'RunningDays': 0, - 'CleanUpDays': 4, + 'CleanUpDays': 0, 'DryRun': 'yes', 'Name': 'Unittest', 'RegionName': 'ap-south-1', 'ResourceStopped': 'False', + 'PublicCloud': 'AWS' } ] assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 diff --git a/tests/unittest/cloud_governance/policy/azure/__init__.py b/tests/unittest/cloud_governance/policy/azure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unittest/cloud_governance/policy/azure/test_vm_run.py b/tests/unittest/cloud_governance/policy/azure/test_vm_run.py new file mode 100644 index 000000000..a39829cce --- /dev/null +++ b/tests/unittest/cloud_governance/policy/azure/test_vm_run.py @@ -0,0 +1,183 @@ +import datetime +from unittest.mock import patch, Mock + +from azure.mgmt.compute import ComputeManagementClient +from azure.mgmt.resource import ResourceManagementClient + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.azure.cleanup.vm_run import VmRun +from tests.unittest.mocks.azure.mock_compute import MockVirtualMachine, MockAzure + + +def test_vm_run(): + """ + This method tests vm_run + :return: + :rtype: + """ + vm1 = MockVirtualMachine(tags={'User': 'mock'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines = Mock() + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines): + vm_run = VmRun() + response = vm_run.run() + assert len(response) == 1 + response = response[0] + assert 'DryRun' in response.keys() + assert 'False' == response['ResourceStopped'] + + +def test_vm_run_stop_false(): + """ + This method tests vm_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'vm_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines): + vm_run = VmRun() + response = vm_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 1 == response[0]['CleanUpDays'] + assert 'False' == response[0]['ResourceStopped'] + + +def test_vm_run_stopped(): + """ + This method tests vm_run + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 0 + environment_variables.environment_variables_dict['policy'] = 'vm_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.begin_deallocate.side_effect = Mock() + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines): + vm_run = VmRun() + response = vm_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 1 == response[0]['CleanUpDays'] + assert 'True' == response[0]['ResourceStopped'] + + +def test_vm_run_stopped_skip(): + """ + This method tests vm_run skip + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 0 + environment_variables.environment_variables_dict['policy'] = 'vm_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock', 'Policy': 'notdelete'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines): + vm_run = VmRun() + response = vm_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() + assert 1 == response[0]['CleanUpDays'] + assert 'False' == response[0]['ResourceStopped'] + + +def test_vm_run_stopped_test_days(): + """ + This method tests vm_run skip + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'vm_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + date = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).date() + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock', 'Policy': 'notdelete', + 'DaysCount': f'{date}@1'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + mock_tags = Mock() + mock_tags.begin_create_or_update_at_scope.side_effect = mock_tags + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines), \ + patch.object(ResourceManagementClient, 'tags', mock_tags): + vm_run = VmRun() + response = vm_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 2 == response[0]['CleanUpDays'] + assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() + assert 'False' == response[0]['ResourceStopped'] + + +def test_vm_run_stopped_test_current_day(): + """ + This method tests vm_run skip + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'vm_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + date = (datetime.datetime.utcnow()).date() + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock', 'Policy': 'notdelete', + 'DaysCount': f'{date}@1'}) + mock_azure = MockAzure(vms=[vm1]) + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + mock_tags = Mock() + mock_tags.begin_create_or_update_at_scope.side_effect = mock_tags + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines), \ + patch.object(ResourceManagementClient, 'tags', mock_tags): + vm_run = VmRun() + response = vm_run.run() + assert len(response) == 1 + assert 'DryRun' in response[0].keys() + assert 1 == response[0]['CleanUpDays'] + assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() + assert 'False' == response[0]['ResourceStopped'] + + +def test_vm_run_vm_already_stopped(): + """ + This method tests vm_run already stopped + :return: + :rtype: + """ + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 3 + environment_variables.environment_variables_dict['policy'] = 'vm_run' + environment_variables.environment_variables_dict['dry_run'] = 'no' + date = (datetime.datetime.utcnow()).date() + mock_virtual_machines = Mock() + vm1 = MockVirtualMachine(tags={'User': 'mock', 'Policy': 'notdelete', + 'DaysCount': f'{date}@1'}) + mock_azure = MockAzure(vms=[vm1], status2="Vm Stopped") + mock_virtual_machines.list_all.side_effect = mock_azure.mock_list_all + mock_virtual_machines.instance_view.side_effect = mock_azure.mock_instance_view + mock_tags = Mock() + mock_tags.begin_create_or_update_at_scope.side_effect = mock_tags + with patch.object(ComputeManagementClient, 'virtual_machines', mock_virtual_machines), \ + patch.object(ResourceManagementClient, 'tags', mock_tags): + vm_run = VmRun() + response = vm_run.run() + assert len(response) == 0 diff --git a/tests/unittest/mocks/__init__.py b/tests/unittest/mocks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unittest/mocks/azure/__init__.py b/tests/unittest/mocks/azure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unittest/mocks/azure/mock_compute.py b/tests/unittest/mocks/azure/mock_compute.py new file mode 100644 index 000000000..c83b0afa1 --- /dev/null +++ b/tests/unittest/mocks/azure/mock_compute.py @@ -0,0 +1,48 @@ +import uuid +from datetime import datetime + +from azure.core.paging import ItemPaged +from azure.mgmt.compute.v2023_03_01.models import VirtualMachine, HardwareProfile, VirtualMachineInstanceView, \ + InstanceViewStatus + + +class MockVirtualMachine(VirtualMachine): + + def __init__(self, tags: dict = None): + super().__init__(location='mock') + self.tags = tags if tags else {} + self.name = 'mock_machine' + self.time_created = datetime.utcnow() + self.hardware_profile = HardwareProfile(vm_size='Standard_D2s_v3') + self.id = f'/subscriptions/{uuid.uuid1()}/resourceGroups/mock/providers/Microsoft.Compute/virtualMachines/mock-machine' + + +class MockVirtualMachineInstanceView(VirtualMachineInstanceView): + + def __init__(self, status1: str = "Unknown", status2: str = 'Vm Running'): + super().__init__() + self.statuses = [ + InstanceViewStatus(display_status=status1), + InstanceViewStatus(display_status=status2) + ] + + +class CustomItemPaged(ItemPaged): + + def __init__(self, vms_list: list = None): + super().__init__() + self._page_iterator = iter(vms_list if vms_list else []) + + +class MockAzure: + + def __init__(self, vms: list = None, status1: str = "Unknown", status2: str = 'Vm Running'): + self.vms = vms if vms else [] + self.status1 = status1 + self.status2 = status2 + + def mock_list_all(self, *args, **kwargs): + return CustomItemPaged(vms_list=self.vms) + + def mock_instance_view(self, *args, **kwargs): + return MockVirtualMachineInstanceView(status1=self.status1, status2=self.status2)