From e9ab307ec7fa6ec6ce136c7ef78302e5a21882cb Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Wed, 27 Mar 2024 13:06:24 +0530 Subject: [PATCH] Added the policy es_model helps to unique policy data --- .../elasticsearch/elasticsearch_operations.py | 5 +- .../common/elasticsearch/modals/__init__.py | 0 .../elasticsearch/modals/policy_es_data.py | 56 +++++++++++++++++++ cloud_governance/common/utils/utils.py | 15 +++++ .../helpers/abstract_policy_operations.py | 44 ++++----------- .../common/utils/test_utils.py | 7 +++ .../policy/aws/cleanup/test_database_idle.py | 4 +- .../policy/aws/cleanup/test_instance_run.py | 17 +++--- .../aws/cleanup/test_unattached_volume.py | 8 +-- .../aws/cleanup/test_unused_nat_gateway.py | 4 +- .../policy/azure/test_instance_run.py | 14 ++--- .../policy/azure/test_unattached_volume.py | 8 +-- 12 files changed, 119 insertions(+), 63 deletions(-) create mode 100644 cloud_governance/common/elasticsearch/modals/__init__.py create mode 100644 cloud_governance/common/elasticsearch/modals/policy_es_data.py diff --git a/cloud_governance/common/elasticsearch/elasticsearch_operations.py b/cloud_governance/common/elasticsearch/elasticsearch_operations.py index 20116574..7f8bddb9 100644 --- a/cloud_governance/common/elasticsearch/elasticsearch_operations.py +++ b/cloud_governance/common/elasticsearch/elasticsearch_operations.py @@ -1,5 +1,4 @@ -import os -from datetime import datetime, timedelta +from datetime import datetime import time import pandas as pd from elasticsearch.helpers import bulk @@ -150,6 +149,8 @@ def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc' data['DryRun'] = self.__environment_variables_dict.get('dry_run') if 'CleanUpDays' not in data: data['CleanUpDays'] = self.__environment_variables_dict.get('DAYS_TO_TAKE_ACTION') + if data.get('IndexId'): + kwargs['id'] = data.get('IndexId') try: if isinstance(data, dict): # JSON Object self.__es.index(index=index, doc_type=doc_type, body=data, **kwargs) diff --git a/cloud_governance/common/elasticsearch/modals/__init__.py b/cloud_governance/common/elasticsearch/modals/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/elasticsearch/modals/policy_es_data.py b/cloud_governance/common/elasticsearch/modals/policy_es_data.py new file mode 100644 index 00000000..da458be7 --- /dev/null +++ b/cloud_governance/common/elasticsearch/modals/policy_es_data.py @@ -0,0 +1,56 @@ +from datetime import datetime, timezone +from dataclasses import dataclass, asdict + +from cloud_governance.common.utils.utils import Utils + + +@dataclass +class PolicyEsMetaData(dict): + + """ + This class is the data modal for policy elasticsearch data + """ + + account: str + resource_id: str + user: str + skip_policy: str + dry_run: str + name: str + region_name: str + public_cloud: str + expire_days: int + + unit_price: float = '' + total_yearly_savings: float = '' + resource_type: str = '' + resource_state: str = '' + clean_up_days: int = '' + days_count: int = '' + resource_action: str = '' + IndexId: str = '' + SnapshotDate: str = datetime.now(timezone.utc).date().__str__() + + # Specific Policy Attributes + volume_size: int = '' + instance_type: str = '' + launch_time: str = '' + running_days: int = '' + create_date: str = '' + + def __post_init__(self): + """ + This method initializes the IndexId + :return: + :rtype: + """ + self.IndexId = (f'{self.SnapshotDate}-{self.public_cloud}-{self.account}-{self.region_name}-{self.resource_id}-' + f'{self.resource_state}').lower() + + def get_as_dict_title_case(self): + """ + This method returns the dict object + :return: + :rtype: + """ + return {Utils.convert_to_title_case(k): v for k, v in asdict(self).items() if v != ''} diff --git a/cloud_governance/common/utils/utils.py b/cloud_governance/common/utils/utils.py index c9a586d9..108bf2fb 100644 --- a/cloud_governance/common/utils/utils.py +++ b/cloud_governance/common/utils/utils.py @@ -2,6 +2,7 @@ import os from datetime import datetime, timedelta from typing import Union +import re class Utils: @@ -130,3 +131,17 @@ def get_start_and_end_datetime(days: int) -> [datetime, datetime]: end_date = datetime.utcnow() start_date = end_date - timedelta(days=days) return start_date, end_date + + @staticmethod + def convert_to_title_case(snake_case: str): + """ + This method converts lower case to title case + ex: test_name => TestName + test-name => TestName + :param snake_case: + :type snake_case: + :return: + :rtype: + """ + title_case = re.sub(r'(?:^|[_-])([a-z])', lambda match: match.group(1).upper(), snake_case) + return title_case diff --git a/cloud_governance/policy/helpers/abstract_policy_operations.py b/cloud_governance/policy/helpers/abstract_policy_operations.py index 5518a2eb..84408c28 100644 --- a/cloud_governance/policy/helpers/abstract_policy_operations.py +++ b/cloud_governance/policy/helpers/abstract_policy_operations.py @@ -6,6 +6,7 @@ from cloud_governance.common.utils.configs import INSTANCE_IDLE_CPU_PERCENTAGE, INSTANCE_IDLE_NETWORK_IN_KILO_BYTES, \ INSTANCE_IDLE_NETWORK_OUT_KILO_BYTES from cloud_governance.common.utils.utils import Utils +from cloud_governance.common.elasticsearch.modals.policy_es_data import PolicyEsMetaData from cloud_governance.main.environment_variables import environment_variables @@ -189,39 +190,16 @@ def __current_savings_year(self, unit_price: float): return total_days * 24 * unit_price # ES Schema format - - def _get_es_schema(self, resource_id: str, user: str, skip_policy: str, cleanup_days: int, dry_run: str, - name: str, - region: str, cleanup_result: str, resource_action: str, cloud_name: str, - resource_state: str, - resource_type: str, **kwargs): - current_date = datetime.utcnow().date() - resource_data = { - 'ResourceId': resource_id, - 'User': user, - 'SkipPolicy': skip_policy, - 'ResourceType': resource_type, - 'ResourceState': resource_state, - 'CleanUpDays': cleanup_days, - 'DryRun': dry_run, - 'Name': name, - 'RegionName': region, - f'Resource{resource_action}': cleanup_result, - 'PublicCloud': cloud_name, - 'ExpireDays': self._days_to_take_action, - 'UnitPrice': kwargs.get('unit_price', 0), - 'TotalYearlySavings': self.__current_savings_year(kwargs.get('unit_price', 0)), - 'index-id': f'{current_date}-{cloud_name.lower()}-{self.account.lower()}-{region.lower()}-{resource_id}-{resource_state.lower()}' - } - if kwargs.get('launch_time'): - resource_data.update({'LaunchTime': kwargs.get('launch_time')}) - if kwargs.get('running_days'): - resource_data.update({'RunningDays': kwargs.get('running_days')}) - if kwargs.get('volume_size'): - resource_data.update({'VolumeSize': kwargs.get('volume_size')}) - if kwargs.get('create_date'): - resource_data.update({'create_date': kwargs.get('create_date')}) - + def _get_es_schema(self, **kwargs): + kwargs['expire_days'] = self._days_to_take_action + kwargs['region_name'] = kwargs.pop('region', '') + kwargs['resource_action'] = kwargs.pop('cleanup_result', '') + kwargs['public_cloud'] = kwargs.pop('cloud_name', '') + kwargs['clean_up_days'] = kwargs.get('cleanup_days', 0) + kwargs['days_count'] = kwargs.pop('cleanup_days', 0) + kwargs['account'] = self.account + policy_es_data = PolicyEsMetaData(**kwargs) + resource_data = policy_es_data.get_as_dict_title_case() return resource_data @abstractmethod diff --git a/tests/unittest/cloud_governance/common/utils/test_utils.py b/tests/unittest/cloud_governance/common/utils/test_utils.py index d57a0b62..4b60b5b0 100644 --- a/tests/unittest/cloud_governance/common/utils/test_utils.py +++ b/tests/unittest/cloud_governance/common/utils/test_utils.py @@ -25,3 +25,10 @@ def test_get_cloud_policies(): from cloud_governance.common.utils.utils import Utils policies = Utils.get_cloud_policies(cloud_name='AWS', dir_dict=True) assert 'instance_run' in policies['cleanup'] + + +def test_convert_to_title_case(): + from cloud_governance.common.utils.utils import Utils + assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test_snake_case') + assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test-snake-case') + assert 'TestSnakeCase' == Utils.convert_to_title_case(snake_case='test_snake-case') diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_database_idle.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_database_idle.py index 20f50676..85370433 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_database_idle.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_database_idle.py @@ -87,7 +87,7 @@ def test_database_idle_delete(): database_idle = DatabaseIdle() running_instances_data = database_idle.run() assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceDelete'] == 'True' + assert running_instances_data[0]['ResourceAction'] == 'True' @mock_cloudwatch @@ -113,6 +113,6 @@ def test_database_idle_dry_run_yes(): database_idle = DatabaseIdle() running_instances_data = database_idle.run() assert running_instances_data[0]['DryRun'] == 'yes' - assert running_instances_data[0]['ResourceDelete'] == 'False' + assert running_instances_data[0]['ResourceAction'] == 'False' assert get_tag_value_from_tags(tags=rds_client.describe_db_instances()['DBInstances'][0]['TagList'], tag_name='DaysCount') == f"{current_date.date()}@0" diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py index 60bc27dd..c2c82873 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_instance_run.py @@ -29,7 +29,7 @@ def test_instance_run(): []) instance_run = InstanceRun() running_instances_data = instance_run.run() - assert running_instances_data[0].get('ResourceStopped') == 'False' + assert running_instances_data[0].get('ResourceAction') == 'False' assert running_instances_data[0].get('ResourceState') == 'running' @@ -59,7 +59,7 @@ def test_instance_run_alert(): assert len(running_instances_data) == 1 assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'False' + assert running_instances_data[0]['ResourceAction'] == 'False' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 @@ -91,7 +91,7 @@ def test_instance_run_alert_stopped(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['ResourceAction'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -122,7 +122,7 @@ def test_instance_run_alert_skip(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'False' + assert running_instances_data[0]['ResourceAction'] == 'False' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 @@ -154,7 +154,7 @@ def test_instance_run_stop_reset(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['ResourceAction'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 instance_run.run() instances = ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["stopped"]}])['Reservations'] @@ -191,7 +191,7 @@ def test_instance_run_stop_start(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['ResourceAction'] == 'True' assert running_instances_data[0]['CleanUpDays'] == 4 assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 instance_run.run() @@ -235,7 +235,7 @@ def test_ec2_force_delete(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'no' - assert running_instances_data[0]['ResourceStopped'] == 'True' + assert running_instances_data[0]['ResourceAction'] == 'True' assert running_instances_data[0]['ForceDeleted'] == 'True' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 0 @@ -269,6 +269,5 @@ def test_ec2_force_delete_skip(): running_instances_data = instance_run.run() assert running_instances_data[0]['ResourceId'] == resource.get('InstanceId') assert running_instances_data[0]['DryRun'] == 'yes' - assert running_instances_data[0]['ResourceStopped'] == 'False' - + assert running_instances_data[0]['ResourceAction'] == 'False' assert len(ec2_client.describe_instances(Filters=[{"Name": "instance-state-name", "Values": ["running"]}])['Reservations']) == 1 diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py index ee3af06c..c53951c8 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_unattached_volume.py @@ -34,7 +34,7 @@ def test_unattached_volume_dry_run_yes(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NA' @@ -51,7 +51,7 @@ def test_unattached_volume_dry_run_no(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'True' + assert response.get('ResourceAction') == 'True' assert response.get('SkipPolicy') == 'NA' @@ -67,7 +67,7 @@ def test_unattached_volume_dry_run_no_7_days_action(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NA' @@ -87,7 +87,7 @@ def test_unattached_volume_dry_run_no_skip(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NOTDELETE' diff --git a/tests/unittest/cloud_governance/policy/aws/cleanup/test_unused_nat_gateway.py b/tests/unittest/cloud_governance/policy/aws/cleanup/test_unused_nat_gateway.py index 4c88019b..ab43b66f 100644 --- a/tests/unittest/cloud_governance/policy/aws/cleanup/test_unused_nat_gateway.py +++ b/tests/unittest/cloud_governance/policy/aws/cleanup/test_unused_nat_gateway.py @@ -103,7 +103,7 @@ def test_unused_nat_gateway___dry_run_no_7_days_action_delete(): assert len(response) == 1 response = response[0] assert response.get('CleanUpDays') == 7 - assert response.get('ResourceDelete') == 'True' + assert response.get('ResourceAction') == 'True' @mock_ec2 @@ -126,7 +126,7 @@ def test_unused_nat_gateway___dry_run_no_skips_delete(): assert len(response) == 1 response = response[0] assert response.get('CleanUpDays') == 7 - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' @mock_ec2 diff --git a/tests/unittest/cloud_governance/policy/azure/test_instance_run.py b/tests/unittest/cloud_governance/policy/azure/test_instance_run.py index c7cb02d6..3f117c66 100644 --- a/tests/unittest/cloud_governance/policy/azure/test_instance_run.py +++ b/tests/unittest/cloud_governance/policy/azure/test_instance_run.py @@ -27,7 +27,7 @@ def test_instance_run(): assert len(response) == 1 response = response[0] assert 'DryRun' in response.keys() - assert 'False' == response['ResourceStopped'] + assert 'False' == response['ResourceAction'] def test_instance_run_stop_false(): @@ -51,7 +51,7 @@ def test_instance_run_stop_false(): assert len(response) == 1 assert 'DryRun' in response[0].keys() assert 1 == response[0]['CleanUpDays'] - assert 'False' == response[0]['ResourceStopped'] + assert 'False' == response[0]['ResourceAction'] def test_instance_run_stopped(): @@ -76,7 +76,7 @@ def test_instance_run_stopped(): assert len(response) == 1 assert 'DryRun' in response[0].keys() assert 1 == response[0]['CleanUpDays'] - assert 'True' == response[0]['ResourceStopped'] + assert 'True' == response[0]['ResourceAction'] def test_instance_run_stopped_skip(): @@ -101,7 +101,7 @@ def test_instance_run_stopped_skip(): assert 'DryRun' in response[0].keys() assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() assert 1 == response[0]['CleanUpDays'] - assert 'False' == response[0]['ResourceStopped'] + assert 'False' == response[0]['ResourceAction'] def test_instance_run_stopped_test_days(): @@ -131,7 +131,7 @@ def test_instance_run_stopped_test_days(): assert 'DryRun' in response[0].keys() assert 2 == response[0]['CleanUpDays'] assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() - assert 'False' == response[0]['ResourceStopped'] + assert 'False' == response[0]['ResourceAction'] def test_instance_run_stopped_test_current_day(): @@ -159,9 +159,9 @@ def test_instance_run_stopped_test_current_day(): response = instance_run.run() assert len(response) == 1 assert 'DryRun' in response[0].keys() - assert 1 == response[0]['CleanUpDays'] + assert 1 == response[0]['DaysCount'] assert 'NOTDELETE' == response[0]['SkipPolicy'].upper() - assert 'False' == response[0]['ResourceStopped'] + assert 'False' == response[0]['ResourceAction'] def test_instance_run_vm_already_stopped(): diff --git a/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py b/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py index 2b66c5cd..efc7b4d3 100644 --- a/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py +++ b/tests/unittest/cloud_governance/policy/azure/test_unattached_volume.py @@ -36,7 +36,7 @@ def test_unattached_volume_dry_run_yes(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NA' @@ -55,7 +55,7 @@ def test_unattached_volume_dry_run_no(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'True' + assert response.get('ResourceAction') == 'True' assert response.get('SkipPolicy') == 'NA' @@ -74,7 +74,7 @@ def test_unattached_volume_dry_run_no_7_days_action(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NA' @@ -94,7 +94,7 @@ def test_unattached_volume_dry_run_no_skip(): response = volume_run.run() assert len(response) > 0 response = response[0] - assert response.get('ResourceDelete') == 'False' + assert response.get('ResourceAction') == 'False' assert response.get('SkipPolicy') == 'NOTDELETE'