Added the azure ec2_run policy

README.md

@@ -22,7 +22,7 @@ This tool support the following policies:
 
 * Real time Openshift Cluster cost, User cost
 * [ec2_idle](cloud_governance/policy/aws/ec2_idle.py): idle ec2 in last 4 days, cpu < 2% & network < 5mb.
-* [ec2_run](cloud_governance/policy/aws/cleanup/ec2_run.py): running ec2.
+* [instance_run](cloud_governance/policy/aws/cleanup/instance_run.py): running ec2.
 * [ebs_unattached](cloud_governance/policy/aws/ebs_unattached.py): volumes that did not connect to instance, volume in available status.
 * [ebs_in_use](cloud_governance/policy/aws/ebs_in_use.py): in use volumes.
 * [tag_resources](cloud_governance/policy/policy_operations/aws/tag_cluster): Update cluster and non cluster resource tags fetching from the user tags or from the mandatory tags
@@ -86,7 +86,7 @@ sudo podman pull quay.io/ebattat/cloud-governance
 (mandatory)AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
 
 ##### Policy name:
-(mandatory)policy=ec2_idle / ec2_run / ebs_unattached / ebs_in_use / tag_cluster_resource / zombie_cluster_resource / tag_ec2_resource
+(mandatory)policy=ec2_idle / instance_run / ebs_unattached / ebs_in_use / tag_cluster_resource / zombie_cluster_resource / tag_ec2_resource
 
 ##### Policy logs output
 (mandatory)policy_output=s3://redhat-cloud-governance/logs
@@ -133,8 +133,8 @@ GOOGLE_APPLICATION_CREDENTIALS=$pwd/service_account.json
 # policy=ec2_idle
 sudo podman run --rm --name cloud-governance -e policy="ec2_idle" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance"
 
-# policy=ec2_run
-sudo podman run --rm --name cloud-governance -e policy="ec2_run" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance"
+# policy=instance_run
+sudo podman run --rm --name cloud-governance -e policy="instance_run" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance"
 
 # select policy ['ec2_stop', 's3_inactive', 'empty_roles', 'ip_unattached', 'unused_nat_gateway', 'zombie_snapshots']
 sudo podman run --rm --name cloud-governance -e policy="policy" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes"  -e log_level="INFO" "quay.io/ebattat/cloud-governance"

cloud_governance/common/clouds/azure/compute/compute_operations.py

@@ -31,9 +31,9 @@ def get_all_instances(self) -> [VirtualMachine]:
         instances_list: [VirtualMachine] = self._item_paged_iterator(item_paged_object=instances_paged_object)
         return instances_list
 
-    def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine:
+    def get_instance_statuses(self, resource_id: str, vm_name: str) -> dict:
         """
-        This method returns the virtual machine data by taking the id
+        This method returns the virtual machine instance status
         :param vm_name:
         :type vm_name:
         :param resource_id:
@@ -42,6 +42,33 @@ def get_instance_data(self, resource_id: str, vm_name: str) -> VirtualMachine:
         :rtype:
         """
         resource_group_name = self._get_resource_group_name_from_resource_id(resource_id=resource_id)
-        virtual_machine = self.__compute_client.virtual_machines.get(resource_group_name=resource_group_name,
-                                                                     vm_name=vm_name)
-        return virtual_machine
+        virtual_machine = self.__compute_client.virtual_machines.instance_view(resource_group_name=resource_group_name,
+                                                                               vm_name=vm_name)
+        return virtual_machine.as_dict()
+
+    def get_id_dict_data(self, resource_id: str):
+        """
+        This method generates the vm id dictionary
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        pairs = resource_id.split('/')[1:]
+        key_pairs = {pairs[i].lower(): pairs[i + 1] for i in range(0, len(pairs), 2)}
+        return key_pairs
+
+    def stop_vm(self, resource_id: str):
+        """
+        This method stops the vm
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        id_key_pairs = self.get_id_dict_data(resource_id)
+        resource_group_name = id_key_pairs.get('resourcegroups')
+        vm_name = id_key_pairs.get('virtualmachines')
+        status = self.__compute_client.virtual_machines.begin_deallocate(resource_group_name=resource_group_name,
+                                                                         vm_name=vm_name)
+        return status.done()

cloud_governance/common/elasticsearch/elasticsearch_operations.py

@@ -144,6 +144,8 @@ def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc'
         # Upload data to elastic search server
         if 'account' not in map(str.lower, data.keys()):
             data['account'] = self.__account
+        if data.get('index-id'):
+            kwargs['id'] = data.get('index-id')
         try:
             if isinstance(data, dict):  # JSON Object
                 self.__es.index(index=index, doc_type=doc_type, body=data, **kwargs)
@@ -310,6 +312,8 @@ def upload_data_in_bulk(self, data_items: list, index: str, **kwargs):
             for item in bulk_items:
                 if kwargs.get('id'):
                     item['_id'] = item.get(kwargs.get('id'))
+                if item.get('index-id'):
+                    item['_id'] = item.get('index-id')
                 if not item.get('timestamp'):
                     if 'CurrentDate' in item:
                         item['timestamp'] = datetime.strptime(item.get('CurrentDate'), "%Y-%m-%d")

cloud_governance/common/helpers/cleanup_operations.py → cloud_governance/common/helpers/abstract_policy_operations.py

@@ -5,7 +5,7 @@
 from cloud_governance.main.environment_variables import environment_variables
 
 
-class AbstractCleanUpOperations(ABC):
+class AbstractPolicyOperations(ABC):
 
     DAYS_TO_NOTIFY_ADMINS = 2
     DAYS_TO_TRIGGER_RESOURCE_MAIL = 4
@@ -14,13 +14,27 @@ class AbstractCleanUpOperations(ABC):
 
     def __init__(self):
         self._environment_variables_dict = environment_variables.environment_variables_dict
+        self.account = self._environment_variables_dict.get('account')
         self._days_to_take_action = self._environment_variables_dict.get('DAYS_TO_TAKE_ACTION')
         self._dry_run = self._environment_variables_dict.get('dry_run')
         self._policy = self._environment_variables_dict.get('policy')
         self._force_delete = self._environment_variables_dict.get('FORCE_DELETE')
         self._resource_id = self._environment_variables_dict.get('RESOURCE_ID')
 
-    @abstractmethod
+    def calculate_days(self, create_date: Union[datetime, str]):
+        """
+        This method returns the days
+        :param create_date:
+        :type create_date:
+        :return:
+        :rtype:
+        """
+        if isinstance(create_date, str):
+            create_date = datetime.strptime(create_date, "%Y-%M-%d")
+        today = datetime.utcnow().date()
+        days = today - create_date.date()
+        return days.days
+
     def get_clean_up_days_count(self, tags: Union[list, dict]):
         """
         This method returns the cleanup days count
@@ -29,7 +43,16 @@ def get_clean_up_days_count(self, tags: Union[list, dict]):
         :return:
         :rtype:
         """
-        raise NotImplementedError("This method is Not yet implemented")
+        if self._dry_run == 'yes':
+            return 0
+        last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount')
+        if not last_used_day:
+            return 1
+        else:
+            date, days = last_used_day.split('@')
+            if date != str(self.CURRENT_DATE):
+                return int(days) + 1
+            return 1 if int(days) == 0 else int(days)
 
     @abstractmethod
     def get_tag_name_from_tags(self, tags: Union[list, dict], tag_name: str):
@@ -85,7 +108,7 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
         """
         raise NotImplementedError("This method is Not yet implemented")
 
-    def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days: int,
+    def verify_and_delete_resource(self, resource_id: str, tags: Union[list, dict], clean_up_days: int,
                                    days_to_delete_resource: int = None, **kwargs):
         """
         This method verify and delete the resource by calculating the days
@@ -109,3 +132,27 @@ def verify_and_delete_resource(self, resource_id: str, tags: list, clean_up_days
                             self._delete_resource(resource_id=resource_id)
                             cleanup_resources = True
         return cleanup_resources
+
+    @abstractmethod
+    def _update_tag_value(self, tags: Union[list, dict], tag_name: str, tag_value: str):
+        """
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :param tag_value:
+        :type tag_value:
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method is Not yet implemented")
+
+    @abstractmethod
+    def _get_al_instances(self):
+        """
+        This method returns all the instances
+        :return:
+        :rtype:
+        """
+        raise NotImplementedError("This method not yet implemented")

cloud_governance/common/helpers/aws/aws_cleanup_operations.py → cloud_governance/common/helpers/aws/aws_policy_operations.py

@@ -1,22 +1,25 @@
-import datetime
 
 import boto3
 
+from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
-from cloud_governance.common.helpers.cleanup_operations import AbstractCleanUpOperations
+from cloud_governance.common.helpers.abstract_policy_operations import AbstractPolicyOperations
 from cloud_governance.common.logger.init_logger import logger
 
 
-class AWSCleanUpOperations(AbstractCleanUpOperations):
+class AWSPolicyOperations(AbstractPolicyOperations):
 
     def __init__(self):
         super().__init__()
         self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
+        self._cloud_name = 'AWS'
         self.__s3operations = S3Operations(region_name=self._region)
         self._ec2_client = boto3.client('ec2', region_name=self._region)
+        self._ec2_operations = EC2Operations(region=self._region)
         self._s3_client = boto3.client('s3')
         self._iam_client = boto3.client('iam')
 
+
     def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
         """
         This method returns the tag value from the tags
@@ -33,23 +36,6 @@ def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
                     return tag.get('Value').strip()
         return ''
 
-    def get_clean_up_days_count(self, tags: list):
-        """
-        This method returns the cleanup days count
-        :param tags:
-        :type tags:
-        :return:
-        :rtype:
-        """
-        last_used_day = self.get_tag_name_from_tags(tags=tags, tag_name='DaysCount')
-        if not last_used_day:
-            return 1
-        else:
-            date, days = last_used_day.split('@')
-            if date != str(self.CURRENT_DATE):
-                return int(days) + 1
-            return 1 if int(days) == 0 else int(days)
-
     def _delete_resource(self, resource_id: str):
         """
         This method deletes the resource by verifying the policy
@@ -72,7 +58,7 @@ def _delete_resource(self, resource_id: str):
                 self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id)
             elif self._policy == 'zombie_snapshots':
                 self._ec2_client.delete_snapshot(SnapshotId=resource_id)
-            elif self._policy == 'ec2_run':
+            elif self._policy == 'instance_run':
                 self._ec2_client.stop_instances(InstanceIds=[resource_id])
                 action = "Stopped"
             logger.info(f'{self._policy} {action}: {resource_id}')
@@ -93,9 +79,9 @@ def __remove_tag_key_aws(self, tags: list):
                 custom_tags.append(tag)
         return custom_tags
 
-    def __update_tag_value(self, tags: list, tag_name: str, tag_value: str):
+    def _update_tag_value(self, tags: list, tag_name: str, tag_value: str):
         """
-        This method updates the tag_value
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
         @param tags:
         @param tag_name:
         @param tag_value:
@@ -133,13 +119,23 @@ def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tag
         :return:
         :rtype:
         """
-        tags = self.__update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
+        tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
         try:
             if self._policy == 's3_inactive':
                 self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})
             elif self._policy == 'empty_roles':
                 self._iam_client.tag_role(RoleName=resource_id, Tags=tags)
-            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached', 'ec2_run'):
+            elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'ebs_unattached',
+                                  'instance_run'):
                 self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
         except Exception as err:
             logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def _get_al_instances(self):
+        """
+        This method updates the instance type count to the elasticsearch
+        :return:
+        :rtype:
+        """
+        instances = self._ec2_operations.get_ec2_instance_list()
+        return instances

cloud_governance/common/helpers/azure/azure_policy_operations.py

@@ -0,0 +1,91 @@
+
+from cloud_governance.common.clouds.azure.compute.compute_operations import ComputeOperations
+from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations
+from cloud_governance.common.helpers.abstract_policy_operations import AbstractPolicyOperations
+from cloud_governance.common.logger.init_logger import logger
+from cloud_governance.common.utils.utils import Utils
+
+
+class AzurePolicyOperations(AbstractPolicyOperations):
+
+    def __init__(self):
+        self._cloud_name = 'Azure'
+        self.compute_operations = ComputeOperations()
+        self.resource_group_operations = ResourceGroupOperations()
+        super().__init__()
+
+    def get_tag_name_from_tags(self, tags: dict, tag_name: str):
+        """
+        This method returns the tag value by the tag_name
+        :param tags:
+        :type tags:
+        :param tag_name:
+        :type tag_name:
+        :return:
+        :rtype:
+        """
+        if tags:
+            for key, value in tags.items():
+                if Utils.equal_ignore_case(key, tag_name):
+                    return value
+        return ''
+
+    def _delete_resource(self, resource_id: str):
+        """
+        This method deletes the
+        :param resource_id:
+        :type resource_id:
+        :return:
+        :rtype:
+        """
+        action = "deleted"
+        try:
+            if self._policy == 'instance_run':
+                action = "Stopped"
+                self.compute_operations.stop_vm(resource_id=resource_id)
+            logger.info(f'{self._policy} {action}: {resource_id}')
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def update_resource_day_count_tag(self, resource_id: str, cleanup_days: int, tags: dict):
+        tags = self._update_tag_value(tags=tags, tag_name='DaysCount', tag_value=str(cleanup_days))
+        try:
+            if self._policy == 'instance_run':
+                self.resource_group_operations.creates_or_updates_tags(resource_id=resource_id, tags=tags)
+        except Exception as err:
+            logger.info(f'Exception raised: {err}: {resource_id}')
+
+    def _update_tag_value(self, tags: dict, tag_name: str, tag_value: str):
+        """
+        This method returns the updated tag_list by adding the tag_name and tag_value to the tags
+        @param tags:
+        @param tag_name:
+        @param tag_value:
+        @return:
+        """
+        if self._dry_run == "yes":
+            tag_value = 0
+        tag_value = f'{self.CURRENT_DATE}@{tag_value}'
+        found = False
+        updated_tags = {}
+        if tags:
+            for key, value in tags.items():
+                if Utils.equal_ignore_case(key, tag_name):
+                    if value.split("@")[0] != self.CURRENT_DATE:
+                        updated_tags[key] = tag_value
+                    else:
+                        if int(tag_value.split("@")[-1]) == 0 or int(tag_value.split("@")[-1]) == 1:
+                            updated_tags[key] = tag_value
+                    found = True
+        tags.update(updated_tags)
+        if not found:
+            tags.update({tag_name: tag_value})
+        return tags
+
+    def _get_al_instances(self):
+        """
+        This method returns the all instances list
+        :return:
+        :rtype:
+        """
+        return self.compute_operations.get_all_instances()