diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index 8faca252..632a3593 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -184,6 +184,7 @@ def __init__(self): self._environment_variables_dict['UPDATE_TAG_BULKS'] = int(EnvironmentVariables.get_env('UPDATE_TAG_BULKS', '20')) # policies aggregate alert + self._environment_variables_dict['SAVE_TO_FILE_PATH'] = EnvironmentVariables.get_env('SAVE_TO_FILE_PATH', '') self._environment_variables_dict['BUCKET_NAME'] = EnvironmentVariables.get_env('BUCKET_NAME') self._environment_variables_dict['BUCKET_KEY'] = EnvironmentVariables.get_env('BUCKET_KEY') self._environment_variables_dict['MAIL_ALERT_DAYS'] = literal_eval(EnvironmentVariables.get_env('MAIL_ALERT_DAYS', '[]')) diff --git a/cloud_governance/main/main.py b/cloud_governance/main/main.py index 2f7c857e..ac0dee2a 100644 --- a/cloud_governance/main/main.py +++ b/cloud_governance/main/main.py @@ -181,7 +181,9 @@ def is_policy_aws(): aws_access_key = environment_variables_dict.get('AWS_ACCESS_KEY_ID', '') aws_secret_key = environment_variables_dict.get('AWS_SECRET_ACCESS_KEY', '') public_cloud_name = environment_variables_dict.get('PUBLIC_CLOUD_NAME', '') - return (aws_access_key and aws_secret_key) or (public_cloud_name.lower() == 'aws') + if aws_secret_key and aws_access_key: + return True + return public_cloud_name.lower() == 'aws' @logger_time_stamp diff --git a/cloud_governance/policy/policy_runners/aws/policy_runner.py b/cloud_governance/policy/policy_runners/aws/policy_runner.py index 0816ccc7..f09f77cb 100644 --- a/cloud_governance/policy/policy_runners/aws/policy_runner.py +++ b/cloud_governance/policy/policy_runners/aws/policy_runner.py @@ -4,6 +4,7 @@ from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations from cloud_governance.common.logger.init_logger import logger +from cloud_governance.policy.policy_runners.aws.upload_s3 import UploadS3 from cloud_governance.policy.policy_runners.common.abstract_policy_runner import AbstractPolicyRunner @@ -11,7 +12,6 @@ class PolicyRunner(AbstractPolicyRunner): def __init__(self): super().__init__() - self.__ec2_operations = EC2Operations() def run(self, source: str = "", upload: bool = True): """ @@ -27,11 +27,14 @@ def run(self, source: str = "", upload: bool = True): logger.info(f'account={self._account}, policy={self._policy}, dry_run={self._dry_run}') zombie_non_cluster_policy_module = importlib.import_module(f'cloud_governance.policy.aws.{source_policy}') + policy_result = [] + ec2_operations = EC2Operations() + upload_to_s3 = UploadS3() for cls in inspect.getmembers(zombie_non_cluster_policy_module, inspect.isclass): if self._policy.replace('_', '').replace('-', '') == cls[0].lower(): active_regions = [self._region] if self._run_active_regions: - active_regions = self.__ec2_operations.get_active_regions() + active_regions = ec2_operations.get_active_regions() logger.info("Running the policy in All AWS active regions") for active_region in active_regions: logger.info(f"Running the {self._policy} in Region: {active_region}") @@ -40,7 +43,10 @@ def run(self, source: str = "", upload: bool = True): if isinstance(response, str): logger.info(f'key: {cls[0]}, Response: {response}') else: + policy_result.extend(response) logger.info(f'key: {cls[0]}, count: {len(response)}, {response}') if upload: self._upload_elastic_search.upload(data=response) - self._upload_to_s3.upload(data=response) + upload_to_s3.upload(data=response) + if self._save_to_file_path: + self.write_to_file(data=policy_result) diff --git a/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py b/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py index 852744a4..63464c26 100644 --- a/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py +++ b/cloud_governance/policy/policy_runners/common/abstract_policy_runner.py @@ -1,7 +1,9 @@ +import os.path from abc import abstractmethod, ABC +from typing import Union +from cloud_governance.common.logger.init_logger import logger from cloud_governance.main.environment_variables import environment_variables -from cloud_governance.policy.policy_runners.aws.upload_s3 import UploadS3 from cloud_governance.policy.policy_runners.elasticsearch.upload_elastic_search import UploadElasticSearch @@ -14,9 +16,43 @@ def __init__(self): self._dry_run = self._environment_variables_dict.get('dry_run', 'yes') self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2') self._run_active_regions = self._environment_variables_dict.get('RUN_ACTIVE_REGIONS') - self._upload_to_s3 = UploadS3() self._upload_elastic_search = UploadElasticSearch() + self._save_to_file_path = self._environment_variables_dict.get('SAVE_TO_FILE_PATH') @abstractmethod def run(self): raise NotImplementedError("This method is not yet implemented") + + def write_to_file(self, data: Union[list, dict]): + """ + This method writes the data to file_path passed by the env SAVE_TO_FILE_PATH + :param data: + :type data: + :return: + :rtype: + """ + if self._save_to_file_path: + if os.path.exists(self._save_to_file_path): + if data: + header_added = False + file_name = f'{self._save_to_file_path}/{self._policy}.csv' + with open(file_name, 'w') as file: + if isinstance(data, list): + for item in data: + if not header_added: + keys = [str(val) for val in list(item.keys())] + ["\n"] + file.write(', '.join(keys)) + header_added = True + values = [str(val) for val in list(item.values())] + ["\n"] + file.write(', '.join(values)) + else: + if isinstance(data, dict): + if not header_added: + keys = [str(val) for val in list(data.keys())] + ["\n"] + file.write(', '.join(keys)) + header_added = True + values = [str(val) for val in list(data.values())] + ["\n"] + file.write(', '.join(values)) + logger.info(f"Written the data into the file_name: {file_name}") + else: + raise FileExistsError(f"FilePath not exists {self._save_to_file_path}") diff --git a/tests/unittest/cloud_governance/policy/policy_runners/common/__init__.py b/tests/unittest/cloud_governance/policy/policy_runners/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/cloud_governance/policy/policy_runners/common/test_abstract_policy_runner.py b/tests/unittest/cloud_governance/policy/policy_runners/common/test_abstract_policy_runner.py new file mode 100644 index 00000000..45923ed8 --- /dev/null +++ b/tests/unittest/cloud_governance/policy/policy_runners/common/test_abstract_policy_runner.py @@ -0,0 +1,21 @@ +import os.path +import tempfile + + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner + + +def test_write_to_file(): + """ + This method writes the data to the file + :return: + :rtype: + """ + with tempfile.TemporaryDirectory() as dir_name: + environment_variables.environment_variables_dict['SAVE_TO_FILE_PATH'] = dir_name + environment_variables.environment_variables_dict['policy'] = 'test' + policy_runner = PolicyRunner() + data = [{"ResourceId": "i-123"}, {"ResourceId": "i-456"}] + policy_runner.write_to_file(data=data) + assert os.path.getsize(f'{dir_name}/test.csv') > 1