Added the policy es_model helps to unique policy data

cloud_governance/common/elasticsearch/elasticsearch_operations.py

@@ -1,5 +1,4 @@
-import os
-from datetime import datetime, timedelta
+from datetime import datetime
 import time
 import pandas as pd
 from elasticsearch.helpers import bulk
@@ -150,6 +149,8 @@ def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc'
             data['DryRun'] = self.__environment_variables_dict.get('dry_run')
         if 'CleanUpDays' not in data:
             data['CleanUpDays'] = self.__environment_variables_dict.get('DAYS_TO_TAKE_ACTION')
+        if data.get('IndexId'):
+            kwargs['id'] = data.get('IndexId')
         try:
             if isinstance(data, dict):  # JSON Object
                 self.__es.index(index=index, doc_type=doc_type, body=data, **kwargs)

cloud_governance/common/elasticsearch/modals/policy_es_data.py

@@ -0,0 +1,52 @@
+from datetime import datetime, timezone
+from dataclasses import dataclass, asdict
+
+from cloud_governance.common.utils.utils import Utils
+
+
+@dataclass
+class PolicyEsMetaData(dict):
+
+    account: str
+    resource_id: str
+    user: str
+    skip_policy: str
+    dry_run: str
+    name: str
+    region_name: str
+    public_cloud: str
+    expire_days: int
+
+    unit_price: float = ''
+    total_yearly_savings: float = ''
+    resource_type: str = ''
+    resource_state: str = ''
+    clean_up_days: int = ''
+    days_count: int = ''
+    resource_action: str = ''
+    IndexId: str = ''
+    SnapshotDate: str = datetime.now(timezone.utc).date().__str__()
+
+    # Specific Policy Attributes
+    volume_size: int = ''
+    instance_type: str = ''
+    launch_time: str = ''
+    running_days: int = ''
+    create_date: str = ''
+
+    def __post_init__(self):
+        """
+        This method initialize the IndexId
+        :return:
+        :rtype:
+        """
+        self.IndexId = (f'{self.SnapshotDate}-{self.public_cloud}-{self.account}-{self.region_name}-{self.resource_id}-'
+                        f'{self.resource_state}').lower()
+
+    def get_as_dict_title_case(self):
+        """
+        This method returns the dict object
+        :return:
+        :rtype:
+        """
+        return {Utils.convert_to_title_case(k): v for k, v in asdict(self).items() if v != ''}

cloud_governance/common/utils/utils.py

@@ -2,6 +2,7 @@
 import os
 from datetime import datetime, timedelta
 from typing import Union
+import re
 
 
 class Utils:
@@ -130,3 +131,17 @@ def get_start_and_end_datetime(days: int) -> [datetime, datetime]:
         end_date = datetime.utcnow()
         start_date = end_date - timedelta(days=days)
         return start_date, end_date
+
+    @staticmethod
+    def convert_to_title_case(snake_case: str):
+        """
+        This method convert lower case to title case
+        ex: test_name => TestName
+            test-name => TestName
+        :param snake_case:
+        :type snake_case:
+        :return:
+        :rtype:
+        """
+        title_case = re.sub(r'(?:^|[_-])([a-z])', lambda match: match.group(1).upper(), snake_case)
+        return title_case

cloud_governance/policy/helpers/abstract_policy_operations.py

@@ -6,6 +6,7 @@
 from cloud_governance.common.utils.configs import INSTANCE_IDLE_CPU_PERCENTAGE, INSTANCE_IDLE_NETWORK_IN_KILO_BYTES, \
     INSTANCE_IDLE_NETWORK_OUT_KILO_BYTES
 from cloud_governance.common.utils.utils import Utils
+from cloud_governance.common.elasticsearch.modals.policy_es_data import PolicyEsMetaData
 from cloud_governance.main.environment_variables import environment_variables
 
 
@@ -189,39 +190,16 @@ def __current_savings_year(self, unit_price: float):
         return total_days * 24 * unit_price
 
     # ES Schema format
-
-    def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str,
-                       name: str,
-                       region: str, cleanup_result: str, resource_action: str, cloud_name: str,
-                       resource_state: str,
-                       resource_type: str, **kwargs):
-        current_date = datetime.utcnow().date()
-        resource_data = {
-            'ResourceId': resource_id,
-            'User': user,
-            'SkipPolicy': skip_policy,
-            'ResourceType': resource_type,
-            'ResourceState': resource_state,
-            'CleanUpDays': cleanup_days,
-            'DryRun': dry_run,
-            'Name': name,
-            'RegionName': region,
-            f'Resource{resource_action}': cleanup_result,
-            'PublicCloud': cloud_name,
-            'ExpireDays': self._days_to_take_action,
-            'UnitPrice': kwargs.get('unit_price', 0),
-            'TotalYearlySavings': self.__current_savings_year(kwargs.get('unit_price', 0)),
-            'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}'
-        }
-        if kwargs.get('launch_time'):
-            resource_data.update({'LaunchTime': kwargs.get('launch_time')})
-        if kwargs.get('running_days'):
-            resource_data.update({'RunningDays': kwargs.get('running_days')})
-        if kwargs.get('volume_size'):
-            resource_data.update({'VolumeSize': kwargs.get('volume_size')})
-        if kwargs.get('create_date'):
-            resource_data.update({'create_date': kwargs.get('create_date')})
-
+    def _get_es_schema(self, **kwargs):
+        kwargs['expire_days'] = self._days_to_take_action
+        kwargs['region_name'] = kwargs.pop('region', '')
+        kwargs['resource_action'] = kwargs.pop('cleanup_result', '')
+        kwargs['public_cloud'] = kwargs.pop('cloud_name', '')
+        kwargs['clean_up_days'] = kwargs.get('cleanup_days', 0)
+        kwargs['days_count'] = kwargs.pop('cleanup_days', 0)
+        kwargs['account'] = self.account
+        policy_es_data = PolicyEsMetaData(**kwargs)
+        resource_data = policy_es_data.get_as_dict_title_case()
         return resource_data
 
     @abstractmethod

tests/unittest/cloud_governance/common/utils/test_utils.py

@@ -25,3 +25,10 @@ def test_get_cloud_policies():
     from cloud_governance.common.utils.utils import Utils
     policies = Utils.get_cloud_policies(cloud_name='AWS', dir_dict=True)
     assert 'instance_run' in policies['cleanup']
+
+
+def test_convert_to_title_case():
+    from cloud_governance.common.utils.utils import Utils
+    assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test_snake_case')
+    assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test-snake-case')
+    assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test_snake-case')

tests/unittest/cloud_governance/policy/aws/cleanup/test_database_idle.py

@@ -87,7 +87,7 @@ def test_database_idle_delete():
     database_idle = DatabaseIdle()
     running_instances_data = database_idle.run()
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceDelete'] == 'True'
+    assert running_instances_data[0]['ResourceAction'] == 'True'
 
 
 @mock_cloudwatch
@@ -113,6 +113,6 @@ def test_database_idle_dry_run_yes():
     database_idle = DatabaseIdle()
     running_instances_data = database_idle.run()
     assert running_instances_data[0]['DryRun'] == 'yes'
-    assert running_instances_data[0]['ResourceDelete'] == 'False'
+    assert running_instances_data[0]['ResourceAction'] == 'False'
     assert get_tag_value_from_tags(tags=rds_client.describe_db_instances()['DBInstances'][0]['TagList'],
                                    tag_name='DaysCount') == f"{current_date.date()}@0"

tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py

@@ -29,7 +29,7 @@ def test_instance_run():
                                                                                                             [])
     instance_run = InstanceRun()
     running_instances_data = instance_run.run()
-    assert running_instances_data[0].get('ResourceStopped') == 'False'
+    assert running_instances_data[0].get('ResourceAction') == 'False'
     assert running_instances_data[0].get('ResourceState') == 'running'
 
 
@@ -59,7 +59,7 @@ def test_instance_run_alert():
     assert len(running_instances_data) == 1
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'False'
+    assert running_instances_data[0]['ResourceAction'] == 'False'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1
 
 
@@ -91,7 +91,7 @@ def test_instance_run_alert_stopped():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'True'
+    assert running_instances_data[0]['ResourceAction'] == 'True'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0
 
 
@@ -122,7 +122,7 @@ def test_instance_run_alert_skip():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'False'
+    assert running_instances_data[0]['ResourceAction'] == 'False'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1
 
 
@@ -154,7 +154,7 @@ def test_instance_run_stop_reset():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'True'
+    assert running_instances_data[0]['ResourceAction'] == 'True'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0
     instance_run.run()
     instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations']
@@ -191,7 +191,7 @@ def test_instance_run_stop_start():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'True'
+    assert running_instances_data[0]['ResourceAction'] == 'True'
     assert running_instances_data[0]['CleanUpDays'] == 4
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0
     instance_run.run()
@@ -235,7 +235,7 @@ def test_ec2_force_delete():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'no'
-    assert running_instances_data[0]['ResourceStopped'] == 'True'
+    assert running_instances_data[0]['ResourceAction'] == 'True'
     assert running_instances_data[0]['ForceDeleted'] == 'True'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0
 
@@ -269,6 +269,5 @@ def test_ec2_force_delete_skip():
     running_instances_data = instance_run.run()
     assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId')
     assert running_instances_data[0]['DryRun'] == 'yes'
-    assert running_instances_data[0]['ResourceStopped'] == 'False'
-
+    assert running_instances_data[0]['ResourceAction'] == 'False'
     assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1

tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py

@@ -34,7 +34,7 @@ def test_unattached_volume_dry_run_yes():
     response = volume_run.run()
     assert len(response) > 0
     response = response[0]
-    assert response.get('ResourceDelete') == 'False'
+    assert response.get('ResourceAction') == 'False'
     assert response.get('SkipPolicy') == 'NA'
 
 
@@ -51,7 +51,7 @@ def test_unattached_volume_dry_run_no():
     response = volume_run.run()
     assert len(response) > 0
     response = response[0]
-    assert response.get('ResourceDelete') == 'True'
+    assert response.get('ResourceAction') == 'True'
     assert response.get('SkipPolicy') == 'NA'
 
 
@@ -67,7 +67,7 @@ def test_unattached_volume_dry_run_no_7_days_action():
     response = volume_run.run()
     assert len(response) > 0
     response = response[0]
-    assert response.get('ResourceDelete') == 'False'
+    assert response.get('ResourceAction') == 'False'
     assert response.get('SkipPolicy') == 'NA'
 
 
@@ -87,7 +87,7 @@ def test_unattached_volume_dry_run_no_skip():
     response = volume_run.run()
     assert len(response) > 0
     response = response[0]
-    assert response.get('ResourceDelete') == 'False'
+    assert response.get('ResourceAction') == 'False'
     assert response.get('SkipPolicy') == 'NOTDELETE'
 
 

tests/unittest/cloud_governance/policy/aws/cleanup/test_unused_nat_gateway.py

@@ -103,7 +103,7 @@ def test_unused_nat_gateway___dry_run_no_7_days_action_delete():
     assert len(response) == 1
     response = response[0]
     assert response.get('CleanUpDays') == 7
-    assert response.get('ResourceDelete') == 'True'
+    assert response.get('ResourceAction') == 'True'
 
 
 @mock_ec2
@@ -126,7 +126,7 @@ def test_unused_nat_gateway___dry_run_no_skips_delete():
     assert len(response) == 1
     response = response[0]
     assert response.get('CleanUpDays') == 7
-    assert response.get('ResourceDelete') == 'False'
+    assert response.get('ResourceAction') == 'False'
 
 
 @mock_ec2

tests/unittest/cloud_governance/policy/azure/test_instance_run.py

@@ -27,7 +27,7 @@ def test_instance_run():
         assert len(response) == 1
         response = response[0]
         assert 'DryRun' in response.keys()
-        assert 'False' == response['ResourceStopped']
+        assert 'False' == response['ResourceAction']
 
 
 def test_instance_run_stop_false():
@@ -51,7 +51,7 @@ def test_instance_run_stop_false():
         assert len(response) == 1
         assert 'DryRun' in response[0].keys()
         assert 1 == response[0]['CleanUpDays']
-        assert 'False' == response[0]['ResourceStopped']
+        assert 'False' == response[0]['ResourceAction']
 
 
 def test_instance_run_stopped():
@@ -76,7 +76,7 @@ def test_instance_run_stopped():
         assert len(response) == 1
         assert 'DryRun' in response[0].keys()
         assert 1 == response[0]['CleanUpDays']
-        assert 'True' == response[0]['ResourceStopped']
+        assert 'True' == response[0]['ResourceAction']
 
 
 def test_instance_run_stopped_skip():
@@ -101,7 +101,7 @@ def test_instance_run_stopped_skip():
         assert 'DryRun' in response[0].keys()
         assert 'NOTDELETE' == response[0]['SkipPolicy'].upper()
         assert 1 == response[0]['CleanUpDays']
-        assert 'False' == response[0]['ResourceStopped']
+        assert 'False' == response[0]['ResourceAction']
 
 
 def test_instance_run_stopped_test_days():
@@ -131,7 +131,7 @@ def test_instance_run_stopped_test_days():
         assert 'DryRun' in response[0].keys()
         assert 2 == response[0]['CleanUpDays']
         assert 'NOTDELETE' == response[0]['SkipPolicy'].upper()
-        assert 'False' == response[0]['ResourceStopped']
+        assert 'False' == response[0]['ResourceAction']
 
 
 def test_instance_run_stopped_test_current_day():
@@ -159,9 +159,9 @@ def test_instance_run_stopped_test_current_day():
         response = instance_run.run()
         assert len(response) == 1
         assert 'DryRun' in response[0].keys()
-        assert 1 == response[0]['CleanUpDays']
+        assert 1 == response[0]['DaysCount']
         assert 'NOTDELETE' == response[0]['SkipPolicy'].upper()
-        assert 'False' == response[0]['ResourceStopped']
+        assert 'False' == response[0]['ResourceAction']
 
 
 def test_instance_run_vm_already_stopped():

tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py

@@ -36,7 +36,7 @@ def test_unattached_volume_dry_run_yes():
         response = volume_run.run()
         assert len(response) > 0
         response = response[0]
-        assert response.get('ResourceDelete') == 'False'
+        assert response.get('ResourceAction') == 'False'
         assert response.get('SkipPolicy') == 'NA'
 
 
@@ -55,7 +55,7 @@ def test_unattached_volume_dry_run_no():
         response = volume_run.run()
         assert len(response) > 0
         response = response[0]
-        assert response.get('ResourceDelete') == 'True'
+        assert response.get('ResourceAction') == 'True'
         assert response.get('SkipPolicy') == 'NA'
 
 
@@ -74,7 +74,7 @@ def test_unattached_volume_dry_run_no_7_days_action():
         response = volume_run.run()
         assert len(response) > 0
         response = response[0]
-        assert response.get('ResourceDelete') == 'False'
+        assert response.get('ResourceAction') == 'False'
         assert response.get('SkipPolicy') == 'NA'
 
 
@@ -94,7 +94,7 @@ def test_unattached_volume_dry_run_no_skip():
         response = volume_run.run()
         assert len(response) > 0
         response = response[0]
-        assert response.get('ResourceDelete') == 'False'
+        assert response.get('ResourceAction') == 'False'
         assert response.get('SkipPolicy') == 'NOTDELETE'