diff --git a/cloud_governance/common/clouds/azure/compute/common_operations.py b/cloud_governance/common/clouds/azure/compute/common_operations.py index b3ce056b..f0054d25 100644 --- a/cloud_governance/common/clouds/azure/compute/common_operations.py +++ b/cloud_governance/common/clouds/azure/compute/common_operations.py @@ -14,7 +14,7 @@ def __init__(self): self._default_creds = DefaultAzureCredential() self._subscription_id = self.__environment_variables_dict.get('AZURE_SUBSCRIPTION_ID') - def _item_paged_iterator(self, item_paged_object: ItemPaged): + def _item_paged_iterator(self, item_paged_object: ItemPaged, as_dict: bool = False): """ This method iterates the paged object and return the list :param item_paged_object: @@ -24,7 +24,10 @@ def _item_paged_iterator(self, item_paged_object: ItemPaged): try: page_item = item_paged_object.next() while page_item: - iterator_list.append(page_item) + if as_dict: + iterator_list.append(page_item.as_dict()) + else: + iterator_list.append(page_item) page_item = item_paged_object.next() except StopIteration: pass diff --git a/cloud_governance/common/clouds/azure/compute/compute_operations.py b/cloud_governance/common/clouds/azure/compute/compute_operations.py index dee906a0..fc3916b9 100644 --- a/cloud_governance/common/clouds/azure/compute/compute_operations.py +++ b/cloud_governance/common/clouds/azure/compute/compute_operations.py @@ -72,3 +72,27 @@ def stop_vm(self, resource_id: str): status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name, vm_name=vm_name) return status.done() + + # volumes -> disks + def get_all_disks(self): + """ + This method returns all the disks + :return: + :rtype: + """ + paged_volumes = self.__compute_client.disks.list() + return self._item_paged_iterator(item_paged_object=paged_volumes, as_dict=True) + + def delete_disk(self, resource_id: str): + """ + This method deletes the disk + :param resource_id: + :type resource_id: + :return: + :rtype: + """ + id_key_pairs = self.get_id_dict_data(resource_id) + resource_group_name = id_key_pairs.get('resourcegroups') + disk_name = id_key_pairs.get('disks') + status = self.__compute_client.disks.begin_delete(resource_group_name=resource_group_name, disk_name=disk_name) + return status.done() diff --git a/cloud_governance/main/main_oerations/main_operations.py b/cloud_governance/main/main_oerations/main_operations.py index d287c2ce..53a92c25 100644 --- a/cloud_governance/main/main_oerations/main_operations.py +++ b/cloud_governance/main/main_oerations/main_operations.py @@ -38,7 +38,7 @@ def run(self): policy_runner = self.get_policy_runner() for policy_type, policies in policies_list.items(): # @Todo support for all the aws policies, currently supports ec2_run as urgent requirement - if self._policy in policies and self._policy in "instance_run": + if self._policy in policies and self._policy in ["instance_run", "unattached_volume"]: policy_runner.run(source=policy_type) return True return False diff --git a/cloud_governance/policy/aws/cleanup/unattached_volume.py b/cloud_governance/policy/aws/cleanup/unattached_volume.py new file mode 100644 index 00000000..f4cf71fe --- /dev/null +++ b/cloud_governance/policy/aws/cleanup/unattached_volume.py @@ -0,0 +1,50 @@ + + +from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations +from cloud_governance.common.utils.utils import Utils + + +class UnattachedVolume(AWSPolicyOperations): + + RESOURCE_ACTION = "Delete" + + def __init__(self): + super().__init__() + + def run_policy_operations(self): + """ + This method returns the list of unattached volumes + :return: + :rtype: + """ + unattached_volumes = [] + available_volumes = self._get_all_volumes() + for volume in available_volumes: + tags = volume.get('Tags', []) + resource_id = volume.get('VolumeId') + cleanup_result = False + if Utils.equal_ignore_case(volume.get('State'), 'available'): + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource(resource_id=resource_id, tags=tags, + clean_up_days=cleanup_days) + resource_data = self._get_es_schema(resource_id=resource_id, + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + skip_policy=self.get_skip_policy_value(tags=tags), + cleanup_days=cleanup_days, dry_run=self._dry_run, + name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'), + region=self._region, + cleanup_result=str(cleanup_result), + resource_action=self.RESOURCE_ACTION, + cloud_name=self._cloud_name, + resource_type=volume.get('VolumeType', ''), + resource_state=volume.get('State') if not cleanup_result else "Deleted", + volume_size=f"{volume.get('Size')} GB" + ) + unattached_volumes.append(resource_data) + else: + cleanup_days = 0 + if not cleanup_result: + self.update_resource_day_count_tag(resource_id=resource_id, cleanup_days=cleanup_days, tags=tags) + + return unattached_volumes + diff --git a/cloud_governance/policy/aws/ebs_unattached.py b/cloud_governance/policy/aws/ebs_unattached.py deleted file mode 100644 index 1fc54d25..00000000 --- a/cloud_governance/policy/aws/ebs_unattached.py +++ /dev/null @@ -1,70 +0,0 @@ -from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy - - -class EbsUnattached(NonClusterZombiePolicy): - """ - This class deletes the ebs unattached if more than 7 days, trigger mail if more than 3 days - add Tag policy=skip/Not_Delete to skip deletion - """ - - def __init__(self): - super().__init__() - - def run(self): - """ - This method list all ebs unattached volumes and delete if it is unattached more than 7 days - @return: - """ - return self.__delete_ebs_unattached() - - def __delete_ebs_unattached(self): - """ - This method list all ebs volumes and delete if it is unattached more than 7 days, trigger mail after 4 days - @return: - """ - volumes = self._ec2_client.describe_volumes(Filters=[{'Name': 'status', 'Values': ['available']}])['Volumes'] - unattached_volumes_data = [] - for volume in volumes: - if not self._check_cluster_tag(tags=volume.get('Tags', [])) or self._get_policy_value(tags=volume.get('Tags')) not in ('NOTDELETE', 'SKIP'): - volume_id = volume.get('VolumeId') - launch_days = self._calculate_days(create_date=volume.get('CreateTime')) - if launch_days >= self.DAYS_TO_DELETE_RESOURCE: - ebs_not_used = False - last_detached_time = self._cloudtrail.get_last_time_accessed(resource_id=volume_id, - event_name='DetachVolume', - start_time=self._start_date, - end_time=self._end_date, - optional_event_name=['CreateVolume', - 'CreateTags']) - if not last_detached_time: - last_detached_time = self._start_date - if last_detached_time: - ebs_not_used = True - last_detached_days = self._calculate_days(create_date=last_detached_time) - ebs_cost = self.resource_pricing.get_ebs_cost(volume_type=volume.get('VolumeType'), volume_size=volume.get('Size'), hours=(self.DAILY_HOURS * last_detached_days)) - delta_cost = 0 - if last_detached_days == self.DAYS_TO_NOTIFY_ADMINS: - delta_cost = self.resource_pricing.get_ebs_cost(volume_type=volume.get('VolumeType'), volume_size=volume.get('Size'), hours=(self.DAILY_HOURS * (self.DAYS_TO_DELETE_RESOURCE - self.DAYS_TO_NOTIFY_ADMINS))) - else: - if last_detached_days == self.DAYS_TO_DELETE_RESOURCE: - delta_cost = self.resource_pricing.get_ebs_cost(volume_type=volume.get('VolumeType'), volume_size=volume.get('Size'), hours=(self.DAILY_HOURS * (self.DAYS_TO_DELETE_RESOURCE - self.DAYS_TO_NOTIFY_ADMINS))) - unused_days = self._get_resource_last_used_days(tags=volume.get('Tags')) - unattached_volumes = self._check_resource_and_delete(resource_name='EBS Volume', - resource_id='VolumeId', - resource_type='CreateVolume', - resource=volume, - empty_days=unused_days, - days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE, - extra_purse=ebs_cost, delta_cost=delta_cost) - if unattached_volumes: - unattached_volumes_data.append({'ResourceId': volume.get('VolumeId'), - 'Name': self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='Name'), - 'User': self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='User'), - 'Days': str(unused_days), - 'Skip': self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='Policy') - }) - else: - unused_days = 0 - self._update_resource_tags(resource_id=volume.get('VolumeId'), tags=volume.get('Tags'), - left_out_days=unused_days, resource_left_out=ebs_not_used) - return unattached_volumes_data diff --git a/cloud_governance/policy/aws/monthly_report.py b/cloud_governance/policy/aws/monthly_report.py index d2b3b203..57a72736 100644 --- a/cloud_governance/policy/aws/monthly_report.py +++ b/cloud_governance/policy/aws/monthly_report.py @@ -33,7 +33,7 @@ def policy_description(self, policy_name: str): policy_descriptions = { 'ec2_stop': 'Delete the stopped instances that are stopped for more than 30 days ', 'ec2_idle': 'stops the idle instances in the last 7 days. ( CPU < 5%, Network < 5k )', - 'ebs_unattached': 'Delete unattached EBS volumes, where the unused days are calculated by the last DeattachedTime', + 'unattached_volume': 'Delete unattached EBS volumes.', 'ip_unattached': 'Delete all the elastic_ips that are unused', 'unused_nat_gateway': ' Delete all unused nat gateways', 'zombie_snapshots': 'Delete all the snapshots which the AMI does not use', diff --git a/cloud_governance/policy/azure/cleanup/unattached_volume.py b/cloud_governance/policy/azure/cleanup/unattached_volume.py new file mode 100644 index 00000000..7867f577 --- /dev/null +++ b/cloud_governance/policy/azure/cleanup/unattached_volume.py @@ -0,0 +1,49 @@ + +from cloud_governance.policy.helpers.azure.azure_policy_operations import AzurePolicyOperations +from cloud_governance.common.utils.utils import Utils + + +class UnattachedVolume(AzurePolicyOperations): + + RESOURCE_ACTION = "Delete" + + def __init__(self): + super().__init__() + + def run_policy_operations(self): + """ + This method returns the list of unattached volumes + :return: + :rtype: + """ + unattached_volumes = [] + available_volumes = self._get_all_volumes() + for volume in available_volumes: + tags = volume.get('tags') + cleanup_result = False + if Utils.equal_ignore_case(volume.get('disk_state'), 'Unattached'): + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource( + resource_id=volume.get('id'), tags=tags, + clean_up_days=cleanup_days) + resource_data = self._get_es_schema(resource_id=volume.get('name'), + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + skip_policy=self.get_skip_policy_value(tags=tags), + cleanup_days=cleanup_days, dry_run=self._dry_run, + name=volume.get('name'), region=volume.get('location'), + cleanup_result=str(cleanup_result), + resource_action=self.RESOURCE_ACTION, + cloud_name=self._cloud_name, + resource_type=f"{volume.get('sku', {}).get('tier')} " + f"{volume.get('sku', {}).get('name')}", + resource_state=volume.get('disk_state') if not cleanup_result else "Deleted", + volume_size=f"{volume.get('disk_size_gb')} GB" + ) + unattached_volumes.append(resource_data) + else: + cleanup_days = 0 + if not cleanup_result: + self.update_resource_day_count_tag(resource_id=volume.get("id"), cleanup_days=cleanup_days, tags=tags) + + return unattached_volumes + diff --git a/cloud_governance/policy/helpers/abstract_policy_operations.py b/cloud_governance/policy/helpers/abstract_policy_operations.py index 85ca3ddc..a374f028 100644 --- a/cloud_governance/policy/helpers/abstract_policy_operations.py +++ b/cloud_governance/policy/helpers/abstract_policy_operations.py @@ -159,14 +159,22 @@ def _get_all_instances(self): """ raise NotImplementedError("This method not yet implemented") - def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str, - name: str, region: str, cleanup_result: str, resource_action: str, cloud_name: str, - resource_state: str, resource_type: str, **kwargs): + @abstractmethod + def _get_all_volumes(self): """ - This method returns the es schema data format + This method returns all the volumes :return: :rtype: """ + raise NotImplementedError("This method not yet implemented") + + # ES Schema format + + def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str, + name: str, + region: str, cleanup_result: str, resource_action: str, cloud_name: str, + resource_state: str, + resource_type: str, **kwargs): current_date = datetime.utcnow().date() resource_data = { 'ResourceId': resource_id, diff --git a/cloud_governance/policy/helpers/aws/aws_policy_operations.py b/cloud_governance/policy/helpers/aws/aws_policy_operations.py index 76d643e1..8a217d55 100644 --- a/cloud_governance/policy/helpers/aws/aws_policy_operations.py +++ b/cloud_governance/policy/helpers/aws/aws_policy_operations.py @@ -49,7 +49,7 @@ def _delete_resource(self, resource_id: str): self._s3_client.delete_bucket(Bucket=resource_id) elif self._policy == 'empty_roles': self._iam_client.delete_role(RoleName=resource_id) - elif self._policy == 'ebs_unattached': + elif self._policy == 'unattached_volume': self._ec2_client.delete_volume(VolumeId=resource_id) elif self._policy == 'ip_unattached': self._ec2_client.release_address(AllocationId=resource_id) @@ -124,7 +124,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags}) elif self._policy == 'empty_roles': self._iam_client.tag_role(RoleName=resource_id, Tags=tags) - elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', + elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'unattached_volume', 'instance_run'): self._ec2_client.create_tags(Resources=[resource_id], Tags=tags) except Exception as err: @@ -141,3 +141,12 @@ def _get_all_instances(self): def run_policy_operations(self): raise NotImplementedError("This method needs to be implemented") + + def _get_all_volumes(self, **kwargs) -> list: + """ + This method returns the all volumes + :return: + :rtype: + """ + volumes = self._ec2_operations.get_volumes(**kwargs) + return volumes diff --git a/cloud_governance/policy/helpers/azure/azure_policy_operations.py b/cloud_governance/policy/helpers/azure/azure_policy_operations.py index b809641e..665ad282 100644 --- a/cloud_governance/policy/helpers/azure/azure_policy_operations.py +++ b/cloud_governance/policy/helpers/azure/azure_policy_operations.py @@ -43,6 +43,8 @@ def _delete_resource(self, resource_id: str): if self._policy == 'instance_run': action = "Stopped" self.compute_operations.stop_vm(resource_id=resource_id) + elif self._policy == 'unattached_volume': + self.compute_operations.delete_disk(resource_id=resource_id) logger.info(f'{self._policy} {action}: {resource_id}') except Exception as err: logger.info(f'Exception raised: {err}: {resource_id}') @@ -50,7 +52,7 @@ def _delete_resource(self, resource_id: str): def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict): tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days)) try: - if self._policy == 'instance_run': + if self._policy in ['instance_run', 'unattached_volume']: self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags) except Exception as err: logger.info(f'Exception raised: {err}: {resource_id}') @@ -63,6 +65,8 @@ def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str): @param tag_value: @return: """ + if not tags: + tags = {} if self._dry_run == "yes": tag_value = 0 tag_value = f'{self.CURRENT_DATE}@{tag_value}' @@ -92,3 +96,12 @@ def _get_all_instances(self): def run_policy_operations(self): raise NotImplementedError("This method needs to be implemented") + + def _get_all_volumes(self) -> list: + """ + This method returns the volumes by state + :return: + :rtype: + """ + volumes = self.compute_operations.get_all_disks() + return volumes diff --git a/cloud_governance/policy/policy_runners/azure/policy_runner.py b/cloud_governance/policy/policy_runners/azure/policy_runner.py index f242b3c5..3a29a942 100644 --- a/cloud_governance/policy/policy_runners/azure/policy_runner.py +++ b/cloud_governance/policy/policy_runners/azure/policy_runner.py @@ -23,6 +23,7 @@ def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: b if isinstance(response, str): logger.info(response) else: + logger.info(response) policy_result.extend(response) self._upload_elastic_search.upload(data=policy_result) return policy_result diff --git a/jenkins/clouds/aws/daily/policies/run_policies.py b/jenkins/clouds/aws/daily/policies/run_policies.py index 91e163ed..e18fe405 100644 --- a/jenkins/clouds/aws/daily/policies/run_policies.py +++ b/jenkins/clouds/aws/daily/policies/run_policies.py @@ -60,6 +60,7 @@ def get_policies(type: str = None): policies.remove('spot_savings_analysis') policies.remove('optimize_resources_report') policies.remove('instance_run') +policies.remove('unattached_volume') es_index_env_var = f'-e es_index={ES_INDEX}' if ES_INDEX else '' @@ -91,10 +92,12 @@ def get_policies(type: str = None): os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PSAP" -e MANAGER_EMAIL_ALERT="False" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PSAP}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-SCALE" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF_SCALE}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + +for new_policy in ['instance_run', 'unattached_volume']: # Run the EC2 run policy -os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-DEPT" -e policy="instance_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e RUN_ACTIVE_REGIONS="True" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") -os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PSAP" -e MANAGER_EMAIL_ALERT="False" -e policy="instance_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e RUN_ACTIVE_REGIONS="True" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PSAP}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") -os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-SCALE" -e policy="instance_run" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e RUN_ACTIVE_REGIONS="True" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF_SCALE}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-DEPT" -e policy="{new_policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e RUN_ACTIVE_REGIONS="True" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PSAP" -e MANAGER_EMAIL_ALERT="False" -e policy="{new_policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e RUN_ACTIVE_REGIONS="True" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PSAP}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system(f"""podman run --rm --name cloud-governance --net="host" -e EMAIL_ALERT="False" -e account="PERF-SCALE" -e policy="{new_policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e RUN_ACTIVE_REGIONS="True" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e account_admin="{account_admin}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {es_index_env_var} -e policy_output="s3://{BUCKET_PERF_SCALE}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") @@ -112,6 +115,7 @@ def get_policies(type: str = None): {'account': 'PSAP', 'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID_DELETE_PSAP, 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY_DELETE_PSAP, 'BUCKET_NAME': BUCKET_PSAP}] policies.remove('ebs_in_use') +policies.append('unattached_volume') remove_polices = ['instance_run', 'ebs_in_use', 'zombie_cluster_resource', 'ec2_idle', 'skipped_resources', 'ec2_stop'] # policies that will not aggregate policies = [policy.replace('_', '-') for policy in policies if policy not in remove_polices] common_input_vars = {'PUBLIC_CLOUD_NAME': 'AWS', 'BUCKET_KEY': 'logs', 'KERBEROS_USERS': f"{special_user_mails}", 'LDAP_HOST_NAME': f"{LDAP_HOST_NAME}", 'log_level': "INFO", 'MAIL_ALERT_DAYS': "[4, 6, 7]", 'POLICY_ACTIONS_DAYS': "[7]", 'POLICIES_TO_ALERT': policies, 'es_host': ES_HOST, 'es_port': ES_PORT} @@ -126,7 +130,7 @@ def get_policies(type: str = None): os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") -os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_index="" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") # # Git-leaks run on github not related to any aws account diff --git a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ebs_unattached.py b/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ebs_unattached.py deleted file mode 100644 index 6fb5cd09..00000000 --- a/tests/unittest/cloud_governance/aws/zombie_non_cluster/test_ebs_unattached.py +++ /dev/null @@ -1,75 +0,0 @@ -import os - -import boto3 -from moto import mock_ec2 - -from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy -from cloud_governance.policy.aws.ebs_unattached import EbsUnattached - -AWS_DEFAULT_REGION = 'us-east-2' - - -@mock_ec2 -def test_ebs_unattached_deleted(): - """ - This method test the ebs volume is deleted - @return: - """ - ec2_client = boto3.client('ec2', region_name=AWS_DEFAULT_REGION) - ec2_client.create_volume(AvailabilityZone='us-east-2a', Size=10) - ebs_unattached = NonClusterZombiePolicy() - ebs_unattached.set_dryrun(value='no') - ebs_unattached.set_policy(value='ebs_unattached') - ebs_unattached.set_region(value=AWS_DEFAULT_REGION) - ebs_unattached.DAYS_TO_TRIGGER_RESOURCE_MAIL = -1 - ebs_unattached._check_resource_and_delete(resource_name='Ebs Volume', - resource_id='VolumeId', - resource_type='CreateVolume', - resource=ec2_client.describe_volumes()['Volumes'][0], - empty_days=0, - days_to_delete_resource=0) - assert 0 == len(ec2_client.describe_volumes()['Volumes']) - - -@mock_ec2 -def test_ebs_unattached_skip_deletion(): - """ - This method skip the ebs unattached deletion - @return: - """ - ec2_client = boto3.client('ec2', region_name=AWS_DEFAULT_REGION) - tags = [{'Key': 'policy', 'Value': 'skip'}] - ec2_client.create_volume(AvailabilityZone='us-east-2a', Size=10, - TagSpecifications=[{'ResourceType': 'volume', 'Tags': tags}]) - ebs_unattached = NonClusterZombiePolicy() - ebs_unattached.set_dryrun(value='no') - ebs_unattached.set_policy(value='ebs_unattached') - ebs_unattached.set_region(value=AWS_DEFAULT_REGION) - ebs_unattached.DAYS_TO_TRIGGER_RESOURCE_MAIL = -1 - ebs_unattached._check_resource_and_delete(resource_name='Ebs Volume', - resource_id='VolumeId', - resource_type='CreateVolume', - resource=ec2_client.describe_volumes()['Volumes'][0], - empty_days=0, - days_to_delete_resource=0) - assert 1 == len(ec2_client.describe_volumes()['Volumes']) - - -@mock_ec2 -def test_skip_live_cluster_volumes(): - """ - This method skips the live cluster volumes - @return: - """ - os.environ['policy'] = 'ebs_unattached' - ec2_client = boto3.client('ec2', region_name=os.environ.get('AWS_DEFAULT_REGION')) - tags = [{'Key': 'Name', 'Value': 'CloudGovernanceTestInstance'}, {'Key': 'User', 'Value': 'cloud-governance'}, - {'Key': 'kubernetes.io/cluster/test', 'Value': 'owned'}] - default_ami_id = 'ami-03cf127a' - ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, MinCount=1, - TagSpecifications=[{'ResourceType': 'instance', 'Tags': tags}]) - ec2_client.create_volume(AvailabilityZone='us-east-2a', Size=10, - TagSpecifications=[{'ResourceType': 'volume', 'Tags': tags}]) - ebs_unattached = EbsUnattached() - ebs_unattached.run() - assert 2 == len(ec2_client.describe_volumes()['Volumes']) diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py new file mode 100644 index 00000000..03af071e --- /dev/null +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py @@ -0,0 +1,93 @@ + +import boto3 +from moto import mock_ec2 + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.aws.cleanup.unattached_volume import UnattachedVolume + + +region_name = 'us-east-2' + + +@mock_ec2 +def test_unattached_volume_dry_run_yes_0_unattached(): + environment_variables.environment_variables_dict['dry_run'] = 'yes' + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + default_ami_id = 'ami-03cf127a' + ec2_client = boto3.client('ec2', region_name=region_name) + resource = ec2_client.run_instances(ImageId=default_ami_id, InstanceType='t2.micro', MaxCount=1, + MinCount=1).get('Instances', [])[0] + volume = ec2_client.create_volume(AvailabilityZone=f'{region_name}a', Size=10) + ec2_client.attach_volume(InstanceId=resource.get('InstanceId'), VolumeId=volume.get('VolumeId'), Device='xvdh') + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) == 0 + + +@mock_ec2 +def test_unattached_volume_dry_run_yes(): + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + environment_variables.environment_variables_dict['dry_run'] = 'yes' + ec2_client = boto3.client('ec2', region_name=region_name) + ec2_client.create_volume(AvailabilityZone=f'{region_name}a', Size=10) + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) > 0 + response = response[0] + assert response.get('ResourceDelete') == 'False' + assert response.get('SkipPolicy') == 'NA' + + +@mock_ec2 +def test_unattached_volume_dry_run_no(): + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 1 + ec2_client = boto3.client('ec2', region_name=region_name) + ec2_client.create_volume(AvailabilityZone=f'{region_name}a', Size=10) + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) > 0 + response = response[0] + assert response.get('ResourceDelete') == 'True' + assert response.get('SkipPolicy') == 'NA' + + +@mock_ec2 +def test_unattached_volume_dry_run_no_7_days_action(): + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + ec2_client = boto3.client('ec2', region_name=region_name) + ec2_client.create_volume(AvailabilityZone=f'{region_name}a', Size=10) + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) > 0 + response = response[0] + assert response.get('ResourceDelete') == 'False' + assert response.get('SkipPolicy') == 'NA' + + +@mock_ec2 +def test_unattached_volume_dry_run_no_skip(): + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = region_name + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + tags = [{'Key': 'Policy', 'Value': 'notdelete'}] + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 1 + ec2_client = boto3.client('ec2', region_name=region_name) + ec2_client.create_volume(AvailabilityZone=f'{region_name}a', Size=10, TagSpecifications=[{ + 'ResourceType': 'volume', + 'Tags': tags + }]) + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) > 0 + response = response[0] + assert response.get('ResourceDelete') == 'False' + assert response.get('SkipPolicy') == 'NOTDELETE' diff --git a/tests/unittest/cloud_governance/policy/azure/test_instance_run.py b/tests/unittest/cloud_governance/policy/azure/test_instance_run.py index 290694ee..a1a23a8b 100644 --- a/tests/unittest/cloud_governance/policy/azure/test_instance_run.py +++ b/tests/unittest/cloud_governance/policy/azure/test_instance_run.py @@ -15,6 +15,7 @@ def test_instance_run(): :return: :rtype: """ + environment_variables.environment_variables_dict['dry_run'] = 'yes' vm1 = MockVirtualMachine(tags={'User': 'mock'}) mock_azure = MockAzure(vms=[vm1]) mock_virtual_machines = Mock() diff --git a/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py b/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py new file mode 100644 index 00000000..842d9c0a --- /dev/null +++ b/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py @@ -0,0 +1,88 @@ +from unittest.mock import Mock, patch + +from azure.mgmt.compute import ComputeManagementClient + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.azure.cleanup.unattached_volume import UnattachedVolume +from tests.unittest.mocks.azure.mock_compute import MockDisk, MockAzure + + +def test_unattached_volume_dry_run_yes_0_unattached(): + environment_variables.environment_variables_dict['dry_run'] = 'yes' + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + mock_disk1 = MockDisk(disk_state='Attached', disk_size_gb=4) + mock_azure = MockAzure(disks=[mock_disk1]) + mock_virtual_machines = Mock() + mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks + with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines): + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) == 0 + + +def test_unattached_volume_dry_run_yes(): + environment_variables.environment_variables_dict['dry_run'] = 'yes' + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + mock_disk1 = MockDisk(disk_state='Unattached', disk_size_gb=4) + mock_azure = MockAzure(disks=[mock_disk1]) + mock_virtual_machines = Mock() + mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks + with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines): + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) > 0 + response = response[0] + assert response.get('ResourceDelete') == 'False' + assert response.get('SkipPolicy') == 'NA' + + +def test_unattached_volume_dry_run_no(): + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 1 + mock_disk1 = MockDisk(disk_state='Unattached', disk_size_gb=4) + mock_azure = MockAzure(disks=[mock_disk1]) + mock_virtual_machines = Mock() + mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks + with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines): + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) > 0 + response = response[0] + assert response.get('ResourceDelete') == 'True' + assert response.get('SkipPolicy') == 'NA' + + +def test_unattached_volume_dry_run_no_7_days_action(): + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + mock_disk1 = MockDisk(disk_state='Unattached', disk_size_gb=4) + mock_azure = MockAzure(disks=[mock_disk1]) + mock_virtual_machines = Mock() + mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks + with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines): + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) > 0 + response = response[0] + assert response.get('ResourceDelete') == 'False' + assert response.get('SkipPolicy') == 'NA' + + +def test_unattached_volume_dry_run_no_skip(): + tags = {'Policy': 'notdelete'} + environment_variables.environment_variables_dict['policy'] = 'unattached_volume' + environment_variables.environment_variables_dict['dry_run'] = 'no' + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 1 + mock_disk1 = MockDisk(disk_state='Unattached', disk_size_gb=4, tags=tags) + mock_azure = MockAzure(disks=[mock_disk1]) + mock_virtual_machines = Mock() + mock_virtual_machines.list.side_effect = mock_azure.mock_list_disks + with patch.object(ComputeManagementClient, 'disks', mock_virtual_machines): + volume_run = UnattachedVolume() + response = volume_run.run() + assert len(response) > 0 + response = response[0] + assert response.get('ResourceDelete') == 'False' + assert response.get('SkipPolicy') == 'NOTDELETE' diff --git a/tests/unittest/mocks/azure/mock_compute.py b/tests/unittest/mocks/azure/mock_compute.py index c83b0afa..d83660e4 100644 --- a/tests/unittest/mocks/azure/mock_compute.py +++ b/tests/unittest/mocks/azure/mock_compute.py @@ -2,6 +2,7 @@ from datetime import datetime from azure.core.paging import ItemPaged +from azure.mgmt.compute.v2023_01_02.models import Disk, DiskSku from azure.mgmt.compute.v2023_03_01.models import VirtualMachine, HardwareProfile, VirtualMachineInstanceView, \ InstanceViewStatus @@ -17,6 +18,17 @@ def __init__(self, tags: dict = None): self.id = f'/subscriptions/{uuid.uuid1()}/resourceGroups/mock/providers/Microsoft.Compute/virtualMachines/mock-machine' +class MockDisk(Disk): + + def __init__(self, disk_state: str, disk_size_gb: int, tags: dict = {}, location: str = 'mock', **kwargs: any): + super().__init__(location=location) + self.tags = tags if tags else {} + self.name = 'mock_disk' + self.sku = DiskSku(name='Standard_LRS', tier='Standard') + self.disk_state = disk_state + self.disk_size_gb = disk_size_gb + + class MockVirtualMachineInstanceView(VirtualMachineInstanceView): def __init__(self, status1: str = "Unknown", status2: str = 'Vm Running'): @@ -29,20 +41,24 @@ def __init__(self, status1: str = "Unknown", status2: str = 'Vm Running'): class CustomItemPaged(ItemPaged): - def __init__(self, vms_list: list = None): + def __init__(self, resource_list: list = None): super().__init__() - self._page_iterator = iter(vms_list if vms_list else []) + self._page_iterator = iter(resource_list if resource_list else []) class MockAzure: - def __init__(self, vms: list = None, status1: str = "Unknown", status2: str = 'Vm Running'): + def __init__(self, vms: list = None, disks: list = [], status1: str = "Unknown", status2: str = 'Vm Running'): self.vms = vms if vms else [] self.status1 = status1 self.status2 = status2 + self.disks = disks if disks else [] def mock_list_all(self, *args, **kwargs): - return CustomItemPaged(vms_list=self.vms) + return CustomItemPaged(resource_list=self.vms) def mock_instance_view(self, *args, **kwargs): return MockVirtualMachineInstanceView(status1=self.status1, status2=self.status2) + + def mock_list_disks(self, *args, **kwargs): + return CustomItemPaged(resource_list=self.disks)