Re arranged the order of policies

cloud_governance/policy/aws/cleanup/instance_run.py

@@ -1,14 +1,40 @@
-import datetime
+from datetime import datetime
 
-from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations
-from cloud_governance.policy.abstract_policies.cleanup.abstractinstance_run import AbstractInstanceRun
+from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations
 
 
-class InstanceRun(AbstractInstanceRun, AWSPolicyOperations):
+class InstanceRun(AWSPolicyOperations):
+
+    INSTANCE_TYPES_ES_INDEX = "cloud-governance-instance-types"
+    RESOURCE_ACTION = "Stopped"
 
     def __init__(self):
         super().__init__()
 
+    def _upload_instance_type_count_to_elastic_search(self):
+        """
+        This method uploads the data to elasticsearch
+        :return:
+        :rtype:
+        """
+        instance_types = self._update_instance_type_count()
+        account = self.account
+        current_day = datetime.utcnow()
+        es_instance_types_data = []
+        for region, instance_types in instance_types.items():
+            for instance_type, instance_type_count in instance_types.items():
+                es_instance_types_data.append({
+                    'instance_type': instance_type,
+                    'instance_count': instance_type_count,
+                    'timestamp': current_day,
+                    'region': region,
+                    'account': account,
+                    'PublicCloud': self._cloud_name,
+                    'index_id': f'{instance_type}-{self._cloud_name.lower()}-{account.lower()}-{region}-{str(current_day.date())}'
+                })
+        self._es_upload.es_upload_data(items=es_instance_types_data, es_index=self.INSTANCE_TYPES_ES_INDEX,
+                                       set_index='index_id')
+
     def _update_instance_type_count(self):
         """
         This method returns the instance types count by region
@@ -20,16 +46,17 @@ def _update_instance_type_count(self):
         for instance in resources:
             instance_type = instance.get('InstanceType')
             instance_types.setdefault(self._region, {}).update(
-                {instance_type: instance_types.get(instance_type, 0) + 1}
+                {instance_type: instance_types.get(self._region).get(instance_type, 0) + 1}
             )
         return instance_types
 
-    def _instance_run(self):
+    def run_policy_operations(self):
         """
         This method returns the running instances
         :return:
         :rtype:
         """
+        self._upload_instance_type_count_to_elastic_search()
         instances = self._get_al_instances()
         running_instances_data = []
         for instance in instances:
@@ -40,18 +67,19 @@ def _instance_run(self):
                 cleanup_result = self.verify_and_delete_resource(
                     resource_id=instance.get('InstanceId'), tags=tags,
                     clean_up_days=cleanup_days)
-                resource_data = self._get_es_data_schema_format(
+                resource_data = self._get_es_schema(
                     resource_id=instance.get('InstanceId'),
                     skip_policy=self.get_skip_policy_value(tags=tags),
                     user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
                     launch_time=instance['LaunchTime'].strftime("%Y-%m-%dT%H:%M:%S+00:00"),
-                    instance_type=instance.get('InstanceType'),
-                    instance_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped',
+                    resource_type=instance.get('InstanceType'),
+                    resource_state=instance.get('State', {}).get('Name') if not cleanup_result else 'stopped',
+                    resource_action=self.RESOURCE_ACTION,
                     running_days=running_days, cleanup_days=cleanup_days,
                     dry_run=self._dry_run,
                     name=self.get_tag_name_from_tags(tags=tags, tag_name='Name'),
                     region=self._region, cleanup_result=str(cleanup_result),
-                    cloud_name=self._cloud_name
+                    cloud_name=self._cloud_name,
                 )
                 if self._force_delete and self._dry_run == 'no':
                     resource_data.update({'ForceDeleted': str(self._force_delete)})

cloud_governance/policy/azure/cleanup/instance_run.py

@@ -1,12 +1,40 @@
-from cloud_governance.common.helpers.azure.azure_policy_operations import AzurePolicyOperations
-from cloud_governance.policy.abstract_policies.cleanup.abstractinstance_run import AbstractInstanceRun
+from datetime import datetime
 
+from cloud_governance.policy.helpers.azure.azure_policy_operations import AzurePolicyOperations
 
-class InstanceRun(AbstractInstanceRun, AzurePolicyOperations):
+
+class InstanceRun(AzurePolicyOperations):
+
+    INSTANCE_TYPES_ES_INDEX = "cloud-governance-instance-types"
+    RESOURCE_ACTION = "Stopped"
 
     def __init__(self):
         super().__init__()
 
+    def _upload_instance_type_count_to_elastic_search(self):
+        """
+        This method uploads the data to elasticsearch
+        :return:
+        :rtype:
+        """
+        instance_types = self._update_instance_type_count()
+        account = self.account
+        current_day = datetime.utcnow()
+        es_instance_types_data = []
+        for region, instance_types in instance_types.items():
+            for instance_type, instance_type_count in instance_types.items():
+                es_instance_types_data.append({
+                    'instance_type': instance_type,
+                    'instance_count': instance_type_count,
+                    'timestamp': current_day,
+                    'region': region,
+                    'account': account,
+                    'PublicCloud': self._cloud_name,
+                    'index_id': f'{instance_type}-{self._cloud_name.lower()}-{account.lower()}-{region}-{str(current_day.date())}'
+                })
+        self._es_upload.es_upload_data(items=es_instance_types_data, es_index=self.INSTANCE_TYPES_ES_INDEX,
+                                       set_index='index_id')
+
     def _update_instance_type_count(self):
         """
         This method returns the instance type count by region
@@ -43,12 +71,13 @@ def __get_instance_status(self, resource_id: str, vm_name: str):
             status = 'Unknown Status'
         return status
 
-    def _instance_run(self):
+    def run_policy_operations(self):
         """
         This method returns the running vms in the AAzure cloud and stops based on the action
         :return:
         :rtype:
         """
+        self._upload_instance_type_count_to_elastic_search()
         vms_list = self._get_al_instances()
         running_vms = []
         for vm in vms_list:
@@ -59,16 +88,17 @@ def _instance_run(self):
                 cleanup_days = self.get_clean_up_days_count(tags=tags)
                 cleanup_result = self.verify_and_delete_resource(resource_id=vm.id, tags=tags,
                                                                  clean_up_days=cleanup_days)
-                resource_data = self._get_es_data_schema_format(
+                resource_data = self._get_es_schema(
                     resource_id=vm.name,
                     skip_policy=self.get_skip_policy_value(tags=tags),
                     user=self.get_tag_name_from_tags(tags=tags, tag_name='User'),
                     launch_time=vm.time_created,
-                    instance_type=vm.hardware_profile.vm_size,
-                    instance_state=status if cleanup_result else 'Vm Stopped',
+                    resource_type=vm.hardware_profile.vm_size,
+                    resource_state=status if cleanup_result else 'Vm Stopped',
                     running_days=running_days, cleanup_days=cleanup_days,
                     dry_run=self._dry_run,
                     name=vm.name,
+                    resource_action=self.RESOURCE_ACTION,
                     region=vm.location, cleanup_result=str(cleanup_result),
                     cloud_name=self._cloud_name
                 )

cloud_governance/common/helpers/abstract_policy_operations.py → cloud_governance/policy/helpers/abstract_policy_operations.py

@@ -2,6 +2,7 @@
 from datetime import datetime
 from typing import Union
 
+from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
 from cloud_governance.main.environment_variables import environment_variables
 
 
@@ -20,6 +21,7 @@ def __init__(self):
         self._policy = self._environment_variables_dict.get('policy')
         self._force_delete = self._environment_variables_dict.get('FORCE_DELETE')
         self._resource_id = self._environment_variables_dict.get('RESOURCE_ID')
+        self._es_upload = ElasticUpload()
 
     def calculate_days(self, create_date: Union[datetime, str]):
         """
@@ -156,3 +158,52 @@ def _get_al_instances(self):
         :rtype:
         """
         raise NotImplementedError("This method not yet implemented")
+
+    def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str,
+                       name: str, region: str, cleanup_result: str, resource_action: str, cloud_name: str,
+                       resource_state: str, resource_type: str, **kwargs):
+        """
+        This method returns the es schema data format
+        :return:
+        :rtype:
+        """
+        current_date = datetime.utcnow().date()
+        resource_data = {
+            'ResourceId': resource_id,
+            'User': user,
+            'SkipPolicy': skip_policy,
+            'ResourceType': resource_type,
+            'ResourceState': resource_state,
+            'CleanUpDays': cleanup_days,
+            'DryRun': dry_run,
+            'Name': name,
+            'RegionName': region,
+            f'Resource{resource_action}': cleanup_result,
+            'PublicCloud': cloud_name,
+            'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}'
+        }
+        if kwargs.get('launch_time'):
+            resource_data.update({'LaunchTime': kwargs.get('launch_time')})
+        if kwargs.get('running_days'):
+            resource_data.update({'RunningDays': kwargs.get('running_days')})
+        if kwargs.get('volume_size'):
+            resource_data.update({'VolumeSize': kwargs.get('volume_size')})
+
+        return resource_data
+
+    @abstractmethod
+    def run_policy_operations(self):
+        """
+        This method returns policy output
+        :return:
+        :rtype:
+        """
+        pass
+
+    def run(self):
+        """
+        This method starts the policy operations
+        :return:
+        :rtype:
+        """
+        return self.run_policy_operations()

cloud_governance/common/helpers/aws/aws_policy_operations.py → cloud_governance/policy/helpers/aws/aws_policy_operations.py

@@ -3,7 +3,7 @@
 
 from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
-from cloud_governance.common.helpers.abstract_policy_operations import AbstractPolicyOperations
+from cloud_governance.policy.helpers.abstract_policy_operations import AbstractPolicyOperations
 from cloud_governance.common.logger.init_logger import logger
 
 
@@ -19,7 +19,6 @@ def __init__(self):
         self._s3_client = boto3.client('s3')
         self._iam_client = boto3.client('iam')
 
-
     def get_tag_name_from_tags(self, tags: list, tag_name: str) -> str:
         """
         This method returns the tag value from the tags
@@ -139,3 +138,6 @@ def _get_al_instances(self):
         """
         instances = self._ec2_operations.get_ec2_instance_list()
         return instances
+
+    def run_policy_operations(self):
+        raise NotImplementedError("This method needs to be implemented")

cloud_governance/common/helpers/azure/azure_policy_operations.py → cloud_governance/policy/helpers/azure/azure_policy_operations.py

@@ -1,7 +1,7 @@
 
 from cloud_governance.common.clouds.azure.compute.compute_operations import ComputeOperations
 from cloud_governance.common.clouds.azure.compute.resource_group_operations import ResourceGroupOperations
-from cloud_governance.common.helpers.abstract_policy_operations import AbstractPolicyOperations
+from cloud_governance.policy.helpers.abstract_policy_operations import AbstractPolicyOperations
 from cloud_governance.common.logger.init_logger import logger
 from cloud_governance.common.utils.utils import Utils
 
@@ -88,4 +88,7 @@ def _get_al_instances(self):
         :return:
         :rtype:
         """
-        return self.compute_operations.get_all_instances()
+        return self.compute_operations.get_all_instances()
+
+    def run_policy_operations(self):
+        raise NotImplementedError("This method needs to be implemented")

cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py

@@ -10,7 +10,7 @@
 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
 from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
 from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations
-from cloud_governance.common.helpers.aws.aws_policy_operations import AWSPolicyOperations
+from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations
 from cloud_governance.common.ldap.ldap_search import LdapSearch
 from cloud_governance.common.logger.init_logger import logger
 from cloud_governance.common.mails.mail_message import MailMessage

cloud_governance/policy/policy_runners/aws/upload_s3.py

@@ -2,7 +2,7 @@
 from typing import Union
 
 from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
-from cloud_governance.common.helpers.json_datetime_encoder import JsonDateTimeEncoder
+from cloud_governance.common.utils.json_datetime_encoder import JsonDateTimeEncoder
 from cloud_governance.common.logger.init_logger import logger
 from cloud_governance.policy.policy_runners.common.abstract_upload import AbstractUpload