Uploaded Cluster data to elasticsearch data

cloud_governance/policy/policy_operations/aws/zombie_cluster/run_zombie_cluster_resources.py

@@ -1,6 +1,10 @@
+from datetime import datetime
 
 import typeguard
 
+from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
+from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations
+from cloud_governance.main.environment_variables import environment_variables
 from cloud_governance.policy.policy_operations.aws.zombie_cluster.zombie_cluster_common_methods import ZombieClusterCommonMethods
 from cloud_governance.common.logger.init_logger import logger
 from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
@@ -125,4 +129,18 @@ def zombie_cluster_resource(delete: bool = False, region: str = 'us-east-2', res
     zombie_cluster_common_methods.send_mails_to_cluster_user(notify_data=notify_data, delete_data=delete_data,
                                                              cluster_data=cluster_data)
     zombie_result['all_cluster_data'] = {'count': len(set(all_cluster_data)), 'data': set(sorted(all_cluster_data))}
+    es_operations = ElasticSearchOperations()
+    if es_operations.check_elastic_search_connection():
+        environment_variables_dict = environment_variables.environment_variables_dict
+        es_index = environment_variables_dict.get('es_index')
+        account = environment_variables_dict.get('account', '')
+        if zombie_result:
+            zombie_result['region_name'] = region
+            zombie_result['account'] = account
+            es_operations.upload_to_elasticsearch(data=zombie_result.copy(), index=es_index)
+            logger.info(f'Uploaded the policy results to elasticsearch index: {es_index}')
+        else:
+            logger.error(f'No data to upload on @{account}  at {datetime.utcnow()}')
+    else:
+        logger.error('ElasticSearch host is not pingable, Please check ')
     return zombie_result

jenkins/clouds/aws/daily/policies/Jenkinsfile

@@ -48,16 +48,11 @@ pipeline {
                  sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
             }
         }
-        stage('Run Policies') {
+        stage('Run Policies, Upload ES and S3 bucket') {
             steps {
                  sh 'python3 jenkins/clouds/aws/daily/policies/run_policies.py'
             }
         }
-        stage('Upload ElasticSearch') {
-            steps {
-                sh 'python3 jenkins/clouds/aws/daily/policies/run_upload_es.py'
-            }
-        }
         stage('Finalize Cleanup') {
             steps {
                  sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''

jenkins/tenant/aws/common/run_policies.py

@@ -28,8 +28,7 @@
 for region in regions:
     for policy in policies_not_in_action:
         os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
-        if policy == 'zombie_cluster_resource':
-            os.system(f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="{account_name}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_doc_type="{es_doc_type}" -e bucket="{s3_bucket}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
+
 
 os.system('echo "Running the CloudGovernance policies with dry_run=no" ')
 os.system(f"echo Polices list: {policies_in_action}")
@@ -39,8 +38,6 @@
             os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
         elif policy not in ('empty_roles', 's3_inactive'):
             os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
-        if policy == 'zombie_cluster_resource':
-            os.system(f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="{account_name}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_doc_type="{es_doc_type}" -e bucket="{s3_bucket}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
 
 
 os.system(f"""echo "Running the tag_iam_user" """)