Added the azure ec2_run policy

cloud_governance/common/clouds/azure/compute/compute_operations.py

@@ -31,9 +31,9 @@ def get_all_instances(self) -> [VirtualMachine]:
         instances_list: [VirtualMachine] = self._item_paged_iterator(item_paged_object=instances_paged_object)
         return instances_list
 
-    def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine:
+    def get_instance_statuses(self, resource_id: str, vm_name: str) -> dict:
         """
-        This method returns the virtual machine data by taking the id
+        This method returns the virtual machine instance status
         :param vm_name:
         :type vm_name:
         :param resource_id:
@@ -42,6 +42,33 @@ def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine:
         :rtype:
         """
         resource_group_name = self._get_resource_group_name_from_resource_id(resource_id=resource_id)
-        virtual_machine = self.__compute_client.virtual_machines.get(resource_group_name=resource_group_name,
-                                                                     vm_name=vm_name)
-        return virtual_machine
+        virtual_machine = self.__compute_client.virtual_machines.instance_view(resource_group_name=resource_group_name,
+                                                                               vm_name=vm_name)
+        return virtual_machine.as_dict()
+
+    def get_id_dict_data(self, resource_id: str):
+        """
+        This method generates the vm id dictionary
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        pairs = resource_id.split('/')[1:]
+        key_pairs = {pairs[i].lower(): pairs[i + 1] for i in range(0, len(pairs), 2)}
+        return key_pairs
+
+    def stop_vm(self, resource_id: str):
+        """
+        This method stops the vm
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        id_key_pairs = self.get_id_dict_data(resource_id)
+        resource_group_name = id_key_pairs.get('resourcegroups')
+        vm_name = id_key_pairs.get('virtualmachines')
+        status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name,
+                                                                         vm_name=vm_name)
+        return status.done()

cloud_governance/common/helpers/aws/aws_cleanup_operations.py

@@ -1,8 +1,9 @@
-import datetime
 
 import boto3
 
+from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
+from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
 from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations
 from cloud_governance.common.logger.init_logger import logger
 
@@ -12,8 +13,11 @@ class AWSCleanUpOperations(AbstractCleanUpOperations):
     def __init__(self):
         super().__init__()
         self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
+        self._cloud_name = 'AWS'
         self.__s3operations = S3Operations(region_name=self._region)
         self._ec2_client = boto3.client('ec2', region_name=self._region)
+        self._ec2_operations = EC2Operations(region=self._region)
+        self._es_upload = ElasticUpload()
         self._s3_client = boto3.client('s3')
         self._iam_client = boto3.client('iam')
 
@@ -33,23 +37,6 @@ def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
                     return tag.get('Value').strip()
         return ''
 
-    def get_clean_up_days_count(self, tags: list):
-        """
-        This method returns the cleanup days count
-        :param tags:
-        :type tags:
-        :return:
-        :rtype:
-        """
-        last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount')
-        if not last_used_day:
-            return 1
-        else:
-            date, days = last_used_day.split('@')
-            if date != str(self.CURRENT_DATE):
-                return int(days) + 1
-            return 1 if int(days) == 0 else int(days)
-
     def _delete_resource(self, resource_id: str):
         """
         This method deletes the resource by verifying the policy
@@ -93,9 +80,9 @@ def __remove_tag_key_aws(self, tags: list):
                 custom_tags.append(tag)
         return custom_tags
 
-    def __update_tag_value(self, tags: list, tag_name: str, tag_value: str):
+    def _update_tag_value(self, tags: list, tag_name: str, tag_value: str):
         """
-        This method updates the tag_value
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
         @param tags:
         @param tag_name:
         @param tag_value:
@@ -133,7 +120,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
         :return:
         :rtype:
         """
-        tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
+        tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
         try:
             if self._policy == 's3_inactive':
                 self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})

cloud_governance/common/helpers/azure/azure_cleanup_operations.py

@@ -0,0 +1,83 @@
+
+from cloud_governance.common.clouds.azure.compute.compute_operations import ComputeOperations
+from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations
+from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations
+from cloud_governance.common.logger.init_logger import logger
+from cloud_governance.common.utils.utils import Utils
+
+
+class AzureCleaUpOperations(AbstractCleanUpOperations):
+
+    def __init__(self):
+        super().__init__()
+        self._cloud_name = 'Azure'
+        self.compute_operations = ComputeOperations()
+        self.resource_group_operations = ResourceGroupOperations()
+
+    def get_tag_name_from_tags(self, tags: dict, tag_name: str):
+        """
+        This method returns the tag value by the tag_name
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :return:
+        :rtype:
+        """
+        if tags:
+            for key, value in tags.items():
+                if Utils.equal_ignore_case(key, tag_name):
+                    return value
+        return ''
+
+    def _delete_resource(self, resource_id: str):
+        """
+        This method deletes the
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        action = "deleted"
+        try:
+            if self._policy == 'vm_run':
+                action = "Stopped"
+                self.compute_operations.stop_vm(resource_id=resource_id)
+            logger.info(f'{self._policy} {action}: {resource_id}')
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict):
+        tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
+        try:
+            if self._policy == 'vm_run':
+                self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags)
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str):
+        """
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
+        @param tags:
+        @param tag_name:
+        @param tag_value:
+        @return:
+        """
+        if self._dry_run == "yes":
+            tag_value = 0
+        tag_value = f'{self.CURRENT_DATE}@{tag_value}'
+        found = False
+        updated_tags = {}
+        if tags:
+            for key, value in tags.items():
+                if Utils.equal_ignore_case(key, tag_name):
+                    if value.split("@")[0] != self.CURRENT_DATE:
+                        updated_tags[key] = tag_value
+                    else:
+                        if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1:
+                            updated_tags[key] = tag_value
+                    found = True
+        tags.update(updated_tags)
+        if not found:
+            return {tag_name: tag_value}
+        return tags

cloud_governance/common/helpers/cleanup_operations.py

@@ -14,13 +14,27 @@ class AbstractCleanUpOperations(ABC):
 
     def __init__(self):
         self._environment_variables_dict = environment_variables.environment_variables_dict
+        self.account = self._environment_variables_dict.get('account')
         self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION')
         self._dry_run = self._environment_variables_dict.get('dry_run')
         self._policy = self._environment_variables_dict.get('policy')
         self._force_delete = self._environment_variables_dict.get('FORCE_DELETE')
         self._resource_id = self._environment_variables_dict.get('RESOURCE_ID')
 
-    @abstractmethod
+    def calculate_days(self, create_date: Union[datetime, str]):
+        """
+        This method returns the days
+        :param create_date:
+        :type create_date:
+        :return:
+        :rtype:
+        """
+        if isinstance(create_date, str):
+            create_date = datetime.strptime(create_date, "%Y-%M-%d")
+        today = datetime.utcnow().date()
+        days = today - create_date.date()
+        return days.days
+
     def get_clean_up_days_count(self, tags: Union[list, dict]):
         """
         This method returns the cleanup days count
@@ -29,7 +43,16 @@ def get_clean_up_days_count(self, tags: Union[list, dict]):
         :return:
         :rtype:
         """
-        raise NotImplementedError("This method is Not yet implemented")
+        if self._dry_run == 'yes':
+            return 0
+        last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount')
+        if not last_used_day:
+            return 1
+        else:
+            date, days = last_used_day.split('@')
+            if date != str(self.CURRENT_DATE):
+                return int(days) + 1
+            return 1 if int(days) == 0 else int(days)
 
     @abstractmethod
     def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str):
@@ -85,7 +108,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
         """
         raise NotImplementedError("This method is Not yet implemented")
 
-    def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int,
+    def verify_and_delete_resource(self, resource_id: str, tags: Union[list, dict], clean_up_days: int,
                                    days_to_delete_resource: int = None, **kwargs):
         """
         This method verify and delete the resource by calculating the days
@@ -109,3 +132,18 @@ def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days
                             self._delete_resource(resource_id=resource_id)
                             cleanup_resources = True
         return cleanup_resources
+
+    @abstractmethod
+    def _update_tag_value(self, tags: Union[list, dict], tag_name: str, tag_value: str):
+        """
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :param tag_value:
+        :type tag_value:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")

cloud_governance/common/utils/utils.py

@@ -0,0 +1,50 @@
+
+import os
+
+
+class Utils:
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_cloud_policies(cloud_name: str, file_type: str = '.py', dir_dict: bool = False,
+                           exclude_policies: list = None):
+        """
+        This method returns the policies by cloud_name
+        :return:
+        :rtype:
+        """
+        exclude_policies = [] if not exclude_policies else exclude_policies
+        policies_dict = {}
+        policies_list = []
+        policies_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'policy', cloud_name)
+        for (dir_path, _, filenames) in os.walk(policies_path):
+            immediate_parent = dir_path.split("/")[-1]
+            for filename in filenames:
+                if not filename.startswith('__') and filename.endswith(file_type):
+                    filename = os.path.splitext(filename)[0]
+                    if filename not in exclude_policies:
+                        if dir_dict:
+                            policies_dict.setdefault(immediate_parent, []).append(filename)
+                        else:
+                            policies_list.append(filename)
+        return policies_dict if dir_dict else policies_list
+
+    @staticmethod
+    def equal_ignore_case(str1: str, str2: str, *args):
+        """
+        This method returns boolean by comparing equal in-case sensitive all strings
+        :param str1:
+        :type str1:
+        :param str2:
+        :type str2:
+        :param args:
+        :type args:
+        :return:
+        :rtype:
+        """
+        equal = str1.lower() == str2.lower()
+        for val in args:
+            equal = str1.lower() == val.lower() and equal
+        return equal

cloud_governance/main/main.py

@@ -3,9 +3,8 @@
 from ast import literal_eval  # str to dict
 import boto3  # regions
 
-from cloud_governance.cloud_resource_orchestration.monitor.cloud_monitor import CloudMonitor
-from cloud_governance.main.aws_main_operations import AWSMainOperations
 from cloud_governance.main.main_common_operations import run_common_policies
+from cloud_governance.main.main_oerations.main_operations import MainOperations
 from cloud_governance.main.run_cloud_resource_orchestration import run_cloud_resource_orchestration
 from cloud_governance.policy.policy_operations.aws.cost_expenditure.cost_report_policies import CostReportPolicies
 from cloud_governance.policy.policy_operations.azure.azure_policy_runner import AzurePolicyRunner
@@ -206,10 +205,8 @@ def main():
     es_index = environment_variables_dict.get('es_index', '')
     es_doc_type = environment_variables_dict.get('es_doc_type', '')
     bucket = environment_variables_dict.get('bucket', '')
-    response = False
-    if is_policy_aws():
-        aws_main_operations = AWSMainOperations()
-        response = aws_main_operations.run()
+    main_operations = MainOperations()
+    response = main_operations.run()
     if not response:
         if environment_variables_dict.get('COMMON_POLICIES'):
             run_common_policies()