diff --git a/cloud_governance/common/elasticsearch/elasticsearch_operations.py b/cloud_governance/common/elasticsearch/elasticsearch_operations.py index d23b48642..a2e061ce9 100644 --- a/cloud_governance/common/elasticsearch/elasticsearch_operations.py +++ b/cloud_governance/common/elasticsearch/elasticsearch_operations.py @@ -146,6 +146,8 @@ def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc' data['account'] = self.__account if data.get('index-id'): kwargs['id'] = data.get('index-id') + if data.get('IndexId'): + kwargs['id'] = data.get('IndexId') try: if isinstance(data, dict): # JSON Object self.__es.index(index=index, doc_type=doc_type, body=data, **kwargs) diff --git a/cloud_governance/common/elasticsearch/modals/__init__.py b/cloud_governance/common/elasticsearch/modals/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cloud_governance/common/elasticsearch/modals/policy_es_data.py b/cloud_governance/common/elasticsearch/modals/policy_es_data.py new file mode 100644 index 000000000..d082f525c --- /dev/null +++ b/cloud_governance/common/elasticsearch/modals/policy_es_data.py @@ -0,0 +1,49 @@ +import datetime +from dataclasses import dataclass, asdict + +from cloud_governance.common.utils.utils import Utils + + +@dataclass +class PolicyEsMetaData(dict): + + account: str + resource_id: str + user: str + dry_run: str + name: str + region_name: str + public_cloud: str + action_days: str + skip_policy: str + cost_unit: float = '' + yearly_saving: float = '' + resource_type: str = '' + resource_state: str = '' + days_count: int = '' + resource_action: str = '' + IndexId: str = '' + SnapshotDate: str = datetime.datetime.utcnow().date().__str__() + + # Specific Policy Attributes + volume_size: int = '' + instance_type: str = '' + launch_time: str = '' + running_days: int = '' + + def __post_init__(self): + """ + This method initialize the IndexId + :return: + :rtype: + """ + self.IndexId = (f'{self.SnapshotDate}-{self.public_cloud}-{self.account}-{self.region_name}-{self.resource_id}-' + f'{self.resource_state}').lower() + + def get_as_dict_title_case(self): + """ + This method returns the dict object + :return: + :rtype: + """ + return {Utils.convert_to_title_case(k): v for k, v in asdict(self).items() if v != ''} diff --git a/cloud_governance/common/utils/utils.py b/cloud_governance/common/utils/utils.py index 09ddee7fb..0a8d2b4e4 100644 --- a/cloud_governance/common/utils/utils.py +++ b/cloud_governance/common/utils/utils.py @@ -1,5 +1,6 @@ import os +import re class Utils: @@ -49,3 +50,17 @@ def equal_ignore_case(str1: str, str2: str, *args): for val in args: equal = str1.lower() == val.lower() and equal return equal + + @staticmethod + def convert_to_title_case(snake_case: str): + """ + This method convert lower case to title case + ex: test_name => TestName + test-name => TestName + :param snake_case: + :type snake_case: + :return: + :rtype: + """ + title_case = re.sub(r'(?:^|[_-])([a-z])', lambda match: match.group(1).upper(), snake_case) + return title_case diff --git a/cloud_governance/policy/helpers/abstract_policy_operations.py b/cloud_governance/policy/helpers/abstract_policy_operations.py index 1b27a208d..841cd69e8 100644 --- a/cloud_governance/policy/helpers/abstract_policy_operations.py +++ b/cloud_governance/policy/helpers/abstract_policy_operations.py @@ -3,6 +3,7 @@ from typing import Union from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload +from cloud_governance.common.elasticsearch.modals.policy_es_data import PolicyEsMetaData from cloud_governance.main.environment_variables import environment_variables @@ -174,34 +175,15 @@ def _get_all_volumes(self): # ES Schema format - def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str, - name: str, - region: str, cleanup_result: str, resource_action: str, cloud_name: str, - resource_state: str, - resource_type: str, **kwargs): - current_date = datetime.utcnow().date() - resource_data = { - 'ResourceId': resource_id, - 'User': user, - 'SkipPolicy': skip_policy, - 'ResourceType': resource_type, - 'ResourceState': resource_state, - 'CleanUpDays': cleanup_days, - 'DryRun': dry_run, - 'Name': name, - 'RegionName': region, - f'Resource{resource_action}': cleanup_result, - 'PublicCloud': cloud_name, - 'ExpireDays': self._days_to_take_action, - 'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}' - } - if kwargs.get('launch_time'): - resource_data.update({'LaunchTime': kwargs.get('launch_time')}) - if kwargs.get('running_days'): - resource_data.update({'RunningDays': kwargs.get('running_days')}) - if kwargs.get('volume_size'): - resource_data.update({'VolumeSize': kwargs.get('volume_size')}) - + def _get_es_schema(self, **kwargs): + kwargs['action_days'] = self._days_to_take_action + kwargs['region_name'] = kwargs.pop('region', '') + kwargs['days_count'] = kwargs.pop('cleanup_days', 0) + kwargs['resource_action'] = kwargs.pop('cleanup_result', '') + kwargs['public_cloud'] = kwargs.pop('cloud_name', '') + kwargs['account'] = self.account + policy_es_data = PolicyEsMetaData(**kwargs) + resource_data = policy_es_data.get_as_dict_title_case() return resource_data @abstractmethod diff --git a/tests/unittest/cloud_governance/common/utils/test_utils.py b/tests/unittest/cloud_governance/common/utils/test_utils.py index d57a0b628..4b60b5b0d 100644 --- a/tests/unittest/cloud_governance/common/utils/test_utils.py +++ b/tests/unittest/cloud_governance/common/utils/test_utils.py @@ -25,3 +25,10 @@ def test_get_cloud_policies(): from cloud_governance.common.utils.utils import Utils policies = Utils.get_cloud_policies(cloud_name='AWS', dir_dict=True) assert 'instance_run' in policies['cleanup'] + + +def test_convert_to_title_case(): + from cloud_governance.common.utils.utils import Utils + assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test_snake_case') + assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test-snake-case') + assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test_snake-case') diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py index 62a5fbef7..f0e06f8d8 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py @@ -29,7 +29,7 @@ def test_instance_run(): []) instance_run = InstanceRun() running_instances_data = instance_run.run() - assert running_instances_data[0].get('ResourceStopped') == 'False' + assert running_instances_data[0].get('ResourceAction') == 'False' assert running_instances_data[0].get('ResourceState') == 'running' @@ -59,7 +59,7 @@ def test_instance_run_alert(): assert len(running_instances_data) == 1 assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'False' + assert running_instances_data[0]['ResourceAction'] == 'False' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 @@ -91,7 +91,7 @@ def test_instance_run_alert_stopped(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['ResourceAction'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -122,7 +122,7 @@ def test_instance_run_alert_skip(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'False' + assert running_instances_data[0]['ResourceAction'] == 'False' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 @@ -153,7 +153,7 @@ def test_instance_run_stop_reset(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['ResourceAction'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 instance_run.run() instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations'] @@ -189,8 +189,8 @@ def test_instance_run_stop_start(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'True' - assert running_instances_data[0]['CleanUpDays'] == 4 + assert running_instances_data[0]['ResourceAction'] == 'True' + assert running_instances_data[0]['DaysCount'] == 4 assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 instance_run.run() instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations'] @@ -232,7 +232,7 @@ def test_ec2_force_delete(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['ResourceAction'] == 'True' assert running_instances_data[0]['ForceDeleted'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -265,6 +265,6 @@ def test_ec2_force_delete_skip(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'yes' - assert running_instances_data[0]['ResourceStopped'] == 'False' + assert running_instances_data[0]['ResourceAction'] == 'False' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py index ee3af06c2..c53951c89 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py @@ -34,7 +34,7 @@ def test_unattached_volume_dry_run_yes(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NA' @@ -51,7 +51,7 @@ def test_unattached_volume_dry_run_no(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'True' + assert response.get('ResourceAction') == 'True' assert response.get('SkipPolicy') == 'NA' @@ -67,7 +67,7 @@ def test_unattached_volume_dry_run_no_7_days_action(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NA' @@ -87,7 +87,7 @@ def test_unattached_volume_dry_run_no_skip(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NOTDELETE' diff --git a/tests/unittest/cloud_governance/policy/aws/test_ip_unattached.py b/tests/unittest/cloud_governance/policy/aws/test_ip_unattached.py index 29855c074..330bbe9e7 100644 --- a/tests/unittest/cloud_governance/policy/aws/test_ip_unattached.py +++ b/tests/unittest/cloud_governance/policy/aws/test_ip_unattached.py @@ -23,7 +23,7 @@ def test_ip_unattached__verify_count_zero_dry_run_yes(): addresses = ec2_client.describe_addresses()['Addresses'] assert len(addresses) == 1 assert len(response) == 1 - assert response[0]['CleanUpDays'] == 0 + assert response[0]['DaysCount'] == 0 @mock_ec2 @@ -43,7 +43,7 @@ def test_ip_unattached__verify_count_increased_dry_run_no(): addresses = ec2_client.describe_addresses()['Addresses'] assert len(addresses) == 1 assert len(response) == 1 - assert response[0]['CleanUpDays'] == 1 + assert response[0]['DaysCount'] == 1 @mock_ec2 @@ -65,7 +65,7 @@ def test_ip_unattached__verify_not_delete_on_dry_run_yes(): addresses = ec2_client.describe_addresses()['Addresses'] assert len(addresses) == 1 assert len(response) == 1 - assert response[0]['CleanUpDays'] == 0 + assert response[0]['DaysCount'] == 0 @mock_ec2 @@ -110,7 +110,7 @@ def test_ip_unattached__skips_delete_on_dry_run_no(): addresses = ec2_client.describe_addresses()['Addresses'] assert len(addresses) == 1 assert len(response) == 1 - assert response[0]['CleanUpDays'] == 7 + assert response[0]['DaysCount'] == 7 assert response[0]['ResourceState'] == 'disassociated' diff --git a/tests/unittest/cloud_governance/policy/azure/test_instance_run.py b/tests/unittest/cloud_governance/policy/azure/test_instance_run.py index 713325c4b..c49c3d5fe 100644 --- a/tests/unittest/cloud_governance/policy/azure/test_instance_run.py +++ b/tests/unittest/cloud_governance/policy/azure/test_instance_run.py @@ -27,7 +27,7 @@ def test_instance_run(): assert len(response) == 1 response = response[0] assert 'DryRun' in response.keys() - assert 'False' == response['ResourceStopped'] + assert 'False' == response['ResourceAction'] def test_instance_run_stop_false(): @@ -49,8 +49,8 @@ def test_instance_run_stop_false(): response = instance_run.run() assert len(response) == 1 assert 'DryRun' in response[0].keys() - assert 1 == response[0]['CleanUpDays'] - assert 'False' == response[0]['ResourceStopped'] + assert 1 == response[0]['DaysCount'] + assert 'False' == response[0]['ResourceAction'] def test_instance_run_stopped(): @@ -73,8 +73,8 @@ def test_instance_run_stopped(): response = instance_run.run() assert len(response) == 1 assert 'DryRun' in response[0].keys() - assert 1 == response[0]['CleanUpDays'] - assert 'True' == response[0]['ResourceStopped'] + assert 1 == response[0]['DaysCount'] + assert 'True' == response[0]['ResourceAction'] def test_instance_run_stopped_skip(): @@ -97,8 +97,8 @@ def test_instance_run_stopped_skip(): assert len(response) == 1 assert 'DryRun' in response[0].keys() assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() - assert 1 == response[0]['CleanUpDays'] - assert 'False' == response[0]['ResourceStopped'] + assert 1 == response[0]['DaysCount'] + assert 'False' == response[0]['ResourceAction'] def test_instance_run_stopped_test_days(): @@ -125,9 +125,9 @@ def test_instance_run_stopped_test_days(): response = instance_run.run() assert len(response) == 1 assert 'DryRun' in response[0].keys() - assert 2 == response[0]['CleanUpDays'] + assert 2 == response[0]['DaysCount'] assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() - assert 'False' == response[0]['ResourceStopped'] + assert 'False' == response[0]['ResourceAction'] def test_instance_run_stopped_test_current_day(): @@ -154,9 +154,9 @@ def test_instance_run_stopped_test_current_day(): response = instance_run.run() assert len(response) == 1 assert 'DryRun' in response[0].keys() - assert 1 == response[0]['CleanUpDays'] + assert 1 == response[0]['DaysCount'] assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() - assert 'False' == response[0]['ResourceStopped'] + assert 'False' == response[0]['ResourceAction'] def test_instance_run_vm_already_stopped(): diff --git a/tests/unittest/cloud_governance/policy/azure/test_ip_unattached.py b/tests/unittest/cloud_governance/policy/azure/test_ip_unattached.py index 8a87d7c72..a1da2776a 100644 --- a/tests/unittest/cloud_governance/policy/azure/test_ip_unattached.py +++ b/tests/unittest/cloud_governance/policy/azure/test_ip_unattached.py @@ -129,7 +129,7 @@ def test_ip_unattached_dry_run_yes(): ip_unattached = IpUnattached() response = ip_unattached.run() assert len(response) == 1 - assert response[0]['CleanUpDays'] == 0 + assert response[0]['DaysCount'] == 0 @mock_compute @@ -148,7 +148,7 @@ def test_ip_unattached_dryrun_no(): ip_unattached = IpUnattached() response = ip_unattached.run() assert len(response) == 1 - assert response[0]['CleanUpDays'] == 1 + assert response[0]['DaysCount'] == 1 assert response[0]['ResourceState'] == 'disassociated' @@ -169,7 +169,7 @@ def test_ip_unattached_delete(): ip_unattached = IpUnattached() response = ip_unattached.run() assert len(response) == 1 - assert response[0]['CleanUpDays'] == 7 + assert response[0]['DaysCount'] == 7 assert response[0]['ResourceState'] == 'Deleted' @@ -190,7 +190,7 @@ def test_ip_unattached_skips_delete(): ip_unattached = IpUnattached() response = ip_unattached.run() assert len(response) == 1 - assert response[0]['CleanUpDays'] == 7 + assert response[0]['DaysCount'] == 7 assert response[0]['ResourceState'] == 'disassociated' @@ -211,5 +211,5 @@ def test_ip_unattached_set_counter_zero(): ip_unattached = IpUnattached() response = ip_unattached.run() assert len(response) == 1 - assert response[0]['CleanUpDays'] == 0 + assert response[0]['DaysCount'] == 0 assert response[0]['ResourceState'] == 'disassociated' diff --git a/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py b/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py index 2b66c5cdc..efc7b4d30 100644 --- a/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py +++ b/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py @@ -36,7 +36,7 @@ def test_unattached_volume_dry_run_yes(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NA' @@ -55,7 +55,7 @@ def test_unattached_volume_dry_run_no(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'True' + assert response.get('ResourceAction') == 'True' assert response.get('SkipPolicy') == 'NA' @@ -74,7 +74,7 @@ def test_unattached_volume_dry_run_no_7_days_action(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NA' @@ -94,7 +94,7 @@ def test_unattached_volume_dry_run_no_skip(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NOTDELETE'