Add the policy result to file_path (#704)

cloud_governance/main/environment_variables.py

@@ -184,6 +184,7 @@ def __init__(self):
         self._environment_variables_dict['UPDATE_TAG_BULKS'] = int(EnvironmentVariables.get_env('UPDATE_TAG_BULKS', '20'))
 
         # policies aggregate alert
+        self._environment_variables_dict['SAVE_TO_FILE_PATH'] = EnvironmentVariables.get_env('SAVE_TO_FILE_PATH', '')
         self._environment_variables_dict['BUCKET_NAME'] = EnvironmentVariables.get_env('BUCKET_NAME')
         self._environment_variables_dict['BUCKET_KEY'] = EnvironmentVariables.get_env('BUCKET_KEY')
         self._environment_variables_dict['MAIL_ALERT_DAYS'] = literal_eval(EnvironmentVariables.get_env('MAIL_ALERT_DAYS', '[]'))

cloud_governance/main/main.py

@@ -181,7 +181,9 @@ def is_policy_aws():
     aws_access_key = environment_variables_dict.get('AWS_ACCESS_KEY_ID', '')
     aws_secret_key = environment_variables_dict.get('AWS_SECRET_ACCESS_KEY', '')
     public_cloud_name = environment_variables_dict.get('PUBLIC_CLOUD_NAME', '')
-    return (aws_access_key and aws_secret_key) or (public_cloud_name.lower() == 'aws')
+    if aws_secret_key and aws_access_key:
+        return True
+    return public_cloud_name.lower() == 'aws'
 
 
 @logger_time_stamp

cloud_governance/policy/policy_runners/aws/policy_runner.py

@@ -4,14 +4,14 @@
 
 from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
 from cloud_governance.common.logger.init_logger import logger
+from cloud_governance.policy.policy_runners.aws.upload_s3 import UploadS3
 from cloud_governance.policy.policy_runners.common.abstract_policy_runner import AbstractPolicyRunner
 
 
 class PolicyRunner(AbstractPolicyRunner):
 
     def __init__(self):
         super().__init__()
-        self.__ec2_operations = EC2Operations()
 
     def run(self, source: str = "", upload: bool = True):
         """
@@ -27,11 +27,14 @@ def run(self, source: str = "", upload: bool = True):
         logger.info(f'account={self._account}, policy={self._policy}, dry_run={self._dry_run}')
         zombie_non_cluster_policy_module = importlib.import_module(f'cloud_governance.policy.aws.{source_policy}')
 
+        policy_result = []
+        ec2_operations = EC2Operations()
+        upload_to_s3 = UploadS3()
         for cls in inspect.getmembers(zombie_non_cluster_policy_module, inspect.isclass):
             if self._policy.replace('_', '').replace('-', '') == cls[0].lower():
                 active_regions = [self._region]
                 if self._run_active_regions:
-                    active_regions = self.__ec2_operations.get_active_regions()
+                    active_regions = ec2_operations.get_active_regions()
                     logger.info("Running the policy in All AWS active regions")
                 for active_region in active_regions:
                     logger.info(f"Running the {self._policy} in Region: {active_region}")
@@ -40,7 +43,10 @@ def run(self, source: str = "", upload: bool = True):
                     if isinstance(response, str):
                         logger.info(f'key: {cls[0]}, Response: {response}')
                     else:
+                        policy_result.extend(response)
                         logger.info(f'key: {cls[0]}, count: {len(response)}, {response}')
                         if upload:
                             self._upload_elastic_search.upload(data=response)
-                            self._upload_to_s3.upload(data=response)
+                            upload_to_s3.upload(data=response)
+        if self._save_to_file_path:
+            self.write_to_file(data=policy_result)

cloud_governance/policy/policy_runners/common/abstract_policy_runner.py

@@ -1,7 +1,9 @@
+import os.path
 from abc import abstractmethod, ABC
+from typing import Union
 
+from cloud_governance.common.logger.init_logger import logger
 from cloud_governance.main.environment_variables import environment_variables
-from cloud_governance.policy.policy_runners.aws.upload_s3 import UploadS3
 from cloud_governance.policy.policy_runners.elasticsearch.upload_elastic_search import UploadElasticSearch
 
 
@@ -14,9 +16,43 @@ def __init__(self):
         self._dry_run = self._environment_variables_dict.get('dry_run', 'yes')
         self._region = self._environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
         self._run_active_regions = self._environment_variables_dict.get('RUN_ACTIVE_REGIONS')
-        self._upload_to_s3 = UploadS3()
         self._upload_elastic_search = UploadElasticSearch()
+        self._save_to_file_path = self._environment_variables_dict.get('SAVE_TO_FILE_PATH')
 
     @abstractmethod
     def run(self):
         raise NotImplementedError("This method is not yet implemented")
+
+    def write_to_file(self, data: Union[list, dict]):
+        """
+        This method writes the data to file_path passed by the env SAVE_TO_FILE_PATH
+        :param data:
+        :type data:
+        :return:
+        :rtype:
+        """
+        if self._save_to_file_path:
+            if os.path.exists(self._save_to_file_path):
+                if data:
+                    header_added = False
+                    file_name = f'{self._save_to_file_path}/{self._policy}.csv'
+                    with open(file_name, 'w') as file:
+                        if isinstance(data, list):
+                            for item in data:
+                                if not header_added:
+                                    keys = [str(val) for val in list(item.keys())] + ["\n"]
+                                    file.write(', '.join(keys))
+                                    header_added = True
+                                values = [str(val) for val in list(item.values())] + ["\n"]
+                                file.write(', '.join(values))
+                        else:
+                            if isinstance(data, dict):
+                                if not header_added:
+                                    keys = [str(val) for val in list(data.keys())] + ["\n"]
+                                    file.write(', '.join(keys))
+                                    header_added = True
+                                values = [str(val) for val in list(data.values())] + ["\n"]
+                                file.write(', '.join(values))
+                    logger.info(f"Written the data into the file_name: {file_name}")
+            else:
+                raise FileExistsError(f"FilePath not exists {self._save_to_file_path}")

tests/unittest/cloud_governance/policy/policy_runners/common/test_abstract_policy_runner.py

@@ -0,0 +1,21 @@
+import os.path
+import tempfile
+
+
+from cloud_governance.main.environment_variables import environment_variables
+from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner
+
+
+def test_write_to_file():
+    """
+    This method writes the data to the file
+    :return:
+    :rtype:
+    """
+    with tempfile.TemporaryDirectory() as dir_name:
+        environment_variables.environment_variables_dict['SAVE_TO_FILE_PATH'] = dir_name
+        environment_variables.environment_variables_dict['policy'] = 'test'
+        policy_runner = PolicyRunner()
+        data = [{"ResourceId": "i-123"}, {"ResourceId": "i-456"}]
+        policy_runner.write_to_file(data=data)
+        assert os.path.getsize(f'{dir_name}/test.csv') > 1