Added the policy es_model helps to unique policy data

cloud_governance/common/elasticsearch/elasticsearch_operations.py

@@ -146,6 +146,8 @@ def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc'
             data['account'] = self.__account
         if data.get('index-id'):
             kwargs['id'] = data.get('index-id')
+        if data.get('IndexId'):
+            kwargs['id'] = data.get('IndexId')
         try:
             if isinstance(data, dict):  # JSON Object
                 self.__es.index(index=index, doc_type=doc_type, body=data, **kwargs)

cloud_governance/common/elasticsearch/modals/policy_es_data.py

@@ -0,0 +1,49 @@
+import datetime
+from dataclasses import dataclass, asdict
+
+from cloud_governance.common.utils.utils import Utils
+
+
+@dataclass
+class PolicyEsData(dict):
+
+    account: str
+    resource_id: str
+    user: str
+    dry_run: str
+    name: str
+    region_name: str
+    public_cloud: str
+    action_days: str
+    skip_policy: str
+    cost: float = ''
+    savings: float = ''
+    resource_type: str = ''
+    resource_state: str = ''
+    days_count: int = ''
+    resource_action: str = ''
+    IndexId: str = ''
+    SnapshotDate: str = datetime.datetime.utcnow().date().__str__()
+
+    # Specific Policy Attributes
+    volume_size: int = ''
+    instance_type: str = ''
+    launch_time: str = ''
+    running_days: int = ''
+
+    def __post_init__(self):
+        """
+        This method initialize the IndexId
+        :return:
+        :rtype:
+        """
+        self.IndexId = (f'{self.SnapshotDate}-{self.public_cloud}-{self.account}-{self.region_name}-{self.resource_id}-'
+                        f'{self.resource_state}').lower()
+
+    def get_as_dict_title_case(self):
+        """
+        This method returns the dict object
+        :return:
+        :rtype:
+        """
+        return {Utils.convert_to_title_case(k): v for k, v in asdict(self).items() if v != ''}

cloud_governance/common/utils/utils.py

@@ -1,5 +1,6 @@
 
 import os
+import re
 
 
 class Utils:
@@ -49,3 +50,17 @@ def equal_ignore_case(str1: str, str2: str, *args):
         for val in args:
             equal = str1.lower() == val.lower() and equal
         return equal
+
+    @staticmethod
+    def convert_to_title_case(snake_case: str):
+        """
+        This method convert lower case to title case
+        ex: test_name => TestName
+            test-name => TestName
+        :param snake_case:
+        :type snake_case:
+        :return:
+        :rtype:
+        """
+        title_case = re.sub(r'(?:^|[_-])([a-z])', lambda match: match.group(1).upper(), snake_case)
+        return title_case

cloud_governance/policy/helpers/abstract_policy_operations.py

@@ -3,6 +3,7 @@
 from typing import Union
 
 from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
+from cloud_governance.common.elasticsearch.modals.policy_es_data import PolicyEsData
 from cloud_governance.main.environment_variables import environment_variables
 
 
@@ -174,34 +175,15 @@ def _get_all_volumes(self):
 
     # ES Schema format
 
-    def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str,
-                       name: str,
-                       region: str, cleanup_result: str, resource_action: str, cloud_name: str,
-                       resource_state: str,
-                       resource_type: str, **kwargs):
-        current_date = datetime.utcnow().date()
-        resource_data = {
-            'ResourceId': resource_id,
-            'User': user,
-            'SkipPolicy': skip_policy,
-            'ResourceType': resource_type,
-            'ResourceState': resource_state,
-            'CleanUpDays': cleanup_days,
-            'DryRun': dry_run,
-            'Name': name,
-            'RegionName': region,
-            f'Resource{resource_action}': cleanup_result,
-            'PublicCloud': cloud_name,
-            'ExpireDays': self._days_to_take_action,
-            'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}'
-        }
-        if kwargs.get('launch_time'):
-            resource_data.update({'LaunchTime': kwargs.get('launch_time')})
-        if kwargs.get('running_days'):
-            resource_data.update({'RunningDays': kwargs.get('running_days')})
-        if kwargs.get('volume_size'):
-            resource_data.update({'VolumeSize': kwargs.get('volume_size')})
-
+    def _get_es_schema(self, **kwargs):
+        kwargs['action_days'] = self._days_to_take_action
+        kwargs['region_name'] = kwargs.pop('region', '')
+        kwargs['days_count'] = kwargs.pop('cleanup_days', 0)
+        kwargs['resource_action'] = kwargs.pop('cleanup_result', '')
+        kwargs['public_cloud'] = kwargs.pop('cloud_name', '')
+        kwargs['account'] = self.account
+        policy_es_data = PolicyEsData(**kwargs)
+        resource_data = policy_es_data.get_as_dict_title_case()
         return resource_data
 
     @abstractmethod

tests/unittest/cloud_governance/common/utils/test_utils.py

@@ -25,3 +25,10 @@ def test_get_cloud_policies():
     from cloud_governance.common.utils.utils import Utils
     policies = Utils.get_cloud_policies(cloud_name='AWS', dir_dict=True)
     assert 'instance_run' in policies['cleanup']
+
+
+def test_convert_to_title_case():
+    from cloud_governance.common.utils.utils import Utils
+    assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test_snake_case')
+    assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test-snake-case')
+    assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test_snake-case')

tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py

@@ -29,7 +29,7 @@ def test_instance_run():
                                                                                                             [])
     instance_run = InstanceRun()
     running_instances_data = instance_run.run()
-    assert running_instances_data[0].get('ResourceStopped') == 'False'
+    assert running_instances_data[0].get('ResourceAction') == 'False'
     assert running_instances_data[0].get('ResourceState') == 'running'
 
 
@@ -59,7 +59,7 @@ def test_instance_run_alert():
     assert len(running_instances_data) == 1
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'False'
+    assert running_instances_data[0]['ResourceAction'] == 'False'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1
 
 
@@ -91,7 +91,7 @@ def test_instance_run_alert_stopped():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'True'
+    assert running_instances_data[0]['ResourceAction'] == 'True'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0
 
 
@@ -122,7 +122,7 @@ def test_instance_run_alert_skip():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'False'
+    assert running_instances_data[0]['ResourceAction'] == 'False'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1
 
 
@@ -153,7 +153,7 @@ def test_instance_run_stop_reset():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'True'
+    assert running_instances_data[0]['ResourceAction'] == 'True'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0
     instance_run.run()
     instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations']
@@ -189,8 +189,8 @@ def test_instance_run_stop_start():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'True'
-    assert running_instances_data[0]['CleanUpDays'] == 4
+    assert running_instances_data[0]['ResourceAction'] == 'True'
+    assert running_instances_data[0]['DaysCount'] == 4
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0
     instance_run.run()
     instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations']
@@ -232,7 +232,7 @@ def test_ec2_force_delete():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'True'
+    assert running_instances_data[0]['ResourceAction'] == 'True'
     assert running_instances_data[0]['ForceDeleted'] == 'True'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0
 
@@ -265,6 +265,6 @@ def test_ec2_force_delete_skip():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'yes'
-    assert running_instances_data[0]['ResourceStopped'] == 'False'
+    assert running_instances_data[0]['ResourceAction'] == 'False'
 
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1

tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py

@@ -34,7 +34,7 @@ def test_unattached_volume_dry_run_yes():
     response = volume_run.run()
     assert len(response) > 0
     response = response[0]
-    assert response.get('ResourceDelete') == 'False'
+    assert response.get('ResourceAction') == 'False'
     assert response.get('SkipPolicy') == 'NA'
 
 
@@ -51,7 +51,7 @@ def test_unattached_volume_dry_run_no():
     response = volume_run.run()
     assert len(response) > 0
     response = response[0]
-    assert response.get('ResourceDelete') == 'True'
+    assert response.get('ResourceAction') == 'True'
     assert response.get('SkipPolicy') == 'NA'
 
 
@@ -67,7 +67,7 @@ def test_unattached_volume_dry_run_no_7_days_action():
     response = volume_run.run()
     assert len(response) > 0
     response = response[0]
-    assert response.get('ResourceDelete') == 'False'
+    assert response.get('ResourceAction') == 'False'
     assert response.get('SkipPolicy') == 'NA'
 
 
@@ -87,7 +87,7 @@ def test_unattached_volume_dry_run_no_skip():
     response = volume_run.run()
     assert len(response) > 0
     response = response[0]
-    assert response.get('ResourceDelete') == 'False'
+    assert response.get('ResourceAction') == 'False'
     assert response.get('SkipPolicy') == 'NOTDELETE'
 
 

tests/unittest/cloud_governance/policy/azure/test_instance_run.py

@@ -27,7 +27,7 @@ def test_instance_run():
         assert len(response) == 1
         response = response[0]
         assert 'DryRun' in response.keys()
-        assert 'False' == response['ResourceStopped']
+        assert 'False' == response['ResourceAction']
 
 
 def test_instance_run_stop_false():
@@ -49,8 +49,8 @@ def test_instance_run_stop_false():
         response = instance_run.run()
         assert len(response) == 1
         assert 'DryRun' in response[0].keys()
-        assert 1 == response[0]['CleanUpDays']
-        assert 'False' == response[0]['ResourceStopped']
+        assert 1 == response[0]['DaysCount']
+        assert 'False' == response[0]['ResourceAction']
 
 
 def test_instance_run_stopped():
@@ -73,8 +73,8 @@ def test_instance_run_stopped():
         response = instance_run.run()
         assert len(response) == 1
         assert 'DryRun' in response[0].keys()
-        assert 1 == response[0]['CleanUpDays']
-        assert 'True' == response[0]['ResourceStopped']
+        assert 1 == response[0]['DaysCount']
+        assert 'True' == response[0]['ResourceAction']
 
 
 def test_instance_run_stopped_skip():
@@ -97,8 +97,8 @@ def test_instance_run_stopped_skip():
         assert len(response) == 1
         assert 'DryRun' in response[0].keys()
         assert 'NOTDELETE' == response[0]['SkipPolicy'].upper()
-        assert 1 == response[0]['CleanUpDays']
-        assert 'False' == response[0]['ResourceStopped']
+        assert 1 == response[0]['DaysCount']
+        assert 'False' == response[0]['ResourceAction']
 
 
 def test_instance_run_stopped_test_days():
@@ -125,9 +125,9 @@ def test_instance_run_stopped_test_days():
         response = instance_run.run()
         assert len(response) == 1
         assert 'DryRun' in response[0].keys()
-        assert 2 == response[0]['CleanUpDays']
+        assert 2 == response[0]['DaysCount']
         assert 'NOTDELETE' == response[0]['SkipPolicy'].upper()
-        assert 'False' == response[0]['ResourceStopped']
+        assert 'False' == response[0]['ResourceAction']
 
 
 def test_instance_run_stopped_test_current_day():
@@ -154,9 +154,9 @@ def test_instance_run_stopped_test_current_day():
         response = instance_run.run()
         assert len(response) == 1
         assert 'DryRun' in response[0].keys()
-        assert 1 == response[0]['CleanUpDays']
+        assert 1 == response[0]['DaysCount']
         assert 'NOTDELETE' == response[0]['SkipPolicy'].upper()
-        assert 'False' == response[0]['ResourceStopped']
+        assert 'False' == response[0]['ResourceAction']
 
 
 def test_instance_run_vm_already_stopped():

tests/unittest/cloud_governance/policy/azure/test_ip_unattached.py

@@ -129,7 +129,7 @@ def test_ip_unattached_dry_run_yes():
     ip_unattached = IpUnattached()
     response = ip_unattached.run()
     assert len(response) == 1
-    assert response[0]['CleanUpDays'] == 0
+    assert response[0]['DaysCount'] == 0
 
 
 @mock_compute
@@ -148,7 +148,7 @@ def test_ip_unattached_dryrun_no():
     ip_unattached = IpUnattached()
     response = ip_unattached.run()
     assert len(response) == 1
-    assert response[0]['CleanUpDays'] == 1
+    assert response[0]['DaysCount'] == 1
     assert response[0]['ResourceState'] == 'disassociated'
 
 
@@ -169,7 +169,7 @@ def test_ip_unattached_delete():
     ip_unattached = IpUnattached()
     response = ip_unattached.run()
     assert len(response) == 1
-    assert response[0]['CleanUpDays'] == 7
+    assert response[0]['DaysCount'] == 7
     assert response[0]['ResourceState'] == 'Deleted'
 
 
@@ -190,7 +190,7 @@ def test_ip_unattached_skips_delete():
     ip_unattached = IpUnattached()
     response = ip_unattached.run()
     assert len(response) == 1
-    assert response[0]['CleanUpDays'] == 7
+    assert response[0]['DaysCount'] == 7
     assert response[0]['ResourceState'] == 'disassociated'
 
 
@@ -211,5 +211,5 @@ def test_ip_unattached_set_counter_zero():
     ip_unattached = IpUnattached()
     response = ip_unattached.run()
     assert len(response) == 1
-    assert response[0]['CleanUpDays'] == 0
+    assert response[0]['DaysCount'] == 0
     assert response[0]['ResourceState'] == 'disassociated'