From aec69e28ce3eae33215461940dc1696be595713f Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Mon, 26 Feb 2024 11:26:49 +0530 Subject: [PATCH] Added the admin mail list --- .../main/environment_variables.py | 2 +- .../common_policies/send_aggregated_alerts.py | 17 +++++----- jenkins/clouds/aws/daily/policies/Jenkinsfile | 3 +- .../clouds/aws/daily/policies/run_policies.py | 5 +++ jenkins/tenant/aws/common/run_policies.py | 7 ++++ .../aws/ecoeng_01/PolicyJenkinsfileDaily | 33 ++++++++++--------- .../aws/ecoeng_02/PolicyJenkinsfileDaily | 5 ++- .../aws/ecoeng_03/PolicyJenkinsfileDaily | 10 +++++- 8 files changed, 55 insertions(+), 27 deletions(-) diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index 2b634c826..cf4d04836 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -195,7 +195,7 @@ def __init__(self): self._environment_variables_dict['DEFAULT_ADMINS'] = literal_eval(EnvironmentVariables.get_env('DEFAULT_ADMINS', '[]')) self._environment_variables_dict['KERBEROS_USERS'] = literal_eval(EnvironmentVariables.get_env('KERBEROS_USERS', '[]')) self._environment_variables_dict['POLICIES_TO_ALERT'] = literal_eval(EnvironmentVariables.get_env('POLICIES_TO_ALERT', '[]')) - self._environment_variables_dict['ONLY_ADMINS'] = EnvironmentVariables.get_boolean_from_environment('ONLY_ADMINS', False) + self._environment_variables_dict['ADMIN_MAIL_LIST'] = EnvironmentVariables.get_env('ADMIN_MAIL_LIST', '') if self._environment_variables_dict.get('policy') in ['send_aggregated_alerts']: self._environment_variables_dict['COMMON_POLICIES'] = True # CRO -- Cloud Resource Orch diff --git a/cloud_governance/policy/common_policies/send_aggregated_alerts.py b/cloud_governance/policy/common_policies/send_aggregated_alerts.py index 7d67c17e4..bdc542b98 100644 --- a/cloud_governance/policy/common_policies/send_aggregated_alerts.py +++ b/cloud_governance/policy/common_policies/send_aggregated_alerts.py @@ -72,7 +72,6 @@ def __get_es_data(self): } } } - # print(json.dumps(query, indent=4)) records = self.__es_operations.post_query(query=query, es_index=policy_es_index) return [record.get('_source') for record in records] @@ -156,15 +155,16 @@ def __update_delete_days(self, policy_es_data: list): if not record.get('Skip'): record['Skip'] = 'NA' if record.get('Skip') != 'NA': - delete_date = 'skip delete' + delete_date = 'skip_delete' else: delete_date = 'dry_run=yes' alert_user = True if alert_user: - record['DeleteDate'] = delete_date.__str__() - if record.get('policy') in ['empty_roles', 's3_inactive']: - record['RegionName'] = 'us-east-1' - filtered_policy_es_data.append(record) + if delete_date != 'skip_delete' and delete_date != 'dry_run=yes': + record['DeleteDate'] = delete_date.__str__() + if record.get('policy') in ['empty_roles', 's3_inactive']: + record['RegionName'] = 'us-east-1' + filtered_policy_es_data.append(record) except Exception as err: raise err return filtered_policy_es_data @@ -178,11 +178,12 @@ def __send_aggregate_email_by_es_data(self): policy_es_data = self.__get_es_data() policy_es_data = self.__remove_duplicates(policy_es_data=policy_es_data) policy_es_data = self.__update_delete_days(policy_es_data) - if self.__environment_variables.get('ONLY_ADMINS', ''): + if self.__environment_variables.get('ADMIN_MAIL_LIST', ''): + to_mail_list = self.__environment_variables.get('ADMIN_MAIL_LIST', '') group_by_policy = self.__group_by_policy(policy_data=policy_es_data) if group_by_policy: subject, body = self.__mail_message.get_policy_alert_message(policy_data=group_by_policy) - self.__postfix.send_email_postfix(subject=subject, content=body, to=self.__mail_to, cc=[], mime_type='html') + self.__postfix.send_email_postfix(subject=subject, content=body, to=to_mail_list, cc=[], mime_type='html') else: user_policy_data = self.__group_by_user(policy_data=policy_es_data) for user, user_records in user_policy_data.items(): diff --git a/jenkins/clouds/aws/daily/policies/Jenkinsfile b/jenkins/clouds/aws/daily/policies/Jenkinsfile index e826546d2..36d258de0 100644 --- a/jenkins/clouds/aws/daily/policies/Jenkinsfile +++ b/jenkins/clouds/aws/daily/policies/Jenkinsfile @@ -1,4 +1,4 @@ -account = ['perf-dept', 'perfscale', 'psap'] +account = ['perf-dept' : "", 'perfscale': "", 'psap': ""] pipeline { options { disableConcurrentBuilds() @@ -49,6 +49,7 @@ pipeline { string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { env.account_name = "${account[i]}" + env.ADMIN_MAIL_LIST = "${accounts_list[account]}" sh 'python3 jenkins/clouds/aws/daily/policies/run_policies.py' } } diff --git a/jenkins/clouds/aws/daily/policies/run_policies.py b/jenkins/clouds/aws/daily/policies/run_policies.py index 44f51445d..d52a821f1 100644 --- a/jenkins/clouds/aws/daily/policies/run_policies.py +++ b/jenkins/clouds/aws/daily/policies/run_policies.py @@ -18,6 +18,8 @@ GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] GITHUB_TOKEN = os.environ['GITHUB_TOKEN'] +CLOUD_GOVERNANCE_IMAGE = "quay.io/ebattat/cloud-governance:latest" +ADMIN_MAIL_LIST = os.environ.get('ADMIN_MAIL_LIST', '') def get_policies(file_type: str = '.py', exclude_policies: list = None): @@ -137,3 +139,6 @@ def run_policies(policies: list, dry_run: str = 'yes'): region = 'us-east-1' policy = 'gitleaks' run_cmd(f"""podman run --rm --name cloud-governance -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e git_access_token="{GITHUB_TOKEN}" -e git_repo="https://github.com/redhat-performance" -e several_repos="yes" -e policy_output="s3://{s3_bucket}/{LOGS}/$region" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + + +run_cmd(f"""podman run --rm --name cloud-governance --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" {CLOUD_GOVERNANCE_IMAGE}""") \ No newline at end of file diff --git a/jenkins/tenant/aws/common/run_policies.py b/jenkins/tenant/aws/common/run_policies.py index 471d18f8b..8c3f4566b 100644 --- a/jenkins/tenant/aws/common/run_policies.py +++ b/jenkins/tenant/aws/common/run_policies.py @@ -60,6 +60,8 @@ def get_container_cmd(env_dict: dict): ES_PORT = os.environ['ES_PORT'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] +ADMIN_MAIL_LIST = os.environ.get('ADMIN_MAIL_LIST', '') +CLOUD_GOVERNANCE_IMAGE = "quay.io/ebattat/cloud-governance:latest" policies_in_action = os.environ.get('POLICIES_IN_ACTION', []) if isinstance(policies_in_action, str): @@ -110,3 +112,8 @@ def run_policies(policies: list, dry_run: str = 'yes'): run_cmd(f"""echo "Running the tag_iam_user" """) run_cmd(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + + +# Run the AggMail + +run_cmd(f"""podman run --rm --name cloud-governance-haim --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" {CLOUD_GOVERNANCE_IMAGE}""") \ No newline at end of file diff --git a/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily index eaca46739..7b8440dca 100644 --- a/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily @@ -1,4 +1,4 @@ -account = ['partnerlab'] +accounts_list = ["partnerlab": "matt.dorn@redhat.com, jomckenz@redhat.com"] pipeline { options { disableConcurrentBuilds() @@ -39,14 +39,16 @@ pipeline { stage('Run Cost Policies') { steps { script { - for (int i = 0; i < account.size(); ++i) { - echo "Running for account ${account[i].toUpperCase()}" - withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), - string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), - string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { - env.account_name = "${account[i]}" - sh 'python3 jenkins/tenant/aws/common/run_cost_policies.py' - } + for (account in accounts_list.keySet()) { + echo "Running for account ${account.toUpperCase()}" + withCredentials([string(credentialsId: "${account}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account}-s3-bucket", variable: 's3_bucket')]) + { + env.account_name = "${account}" + env.ADMIN_MAIL_LIST = "${accounts_list[account]}" + sh 'python3 jenkins/tenant/aws/common/run_cost_policies.py' + } } } } @@ -54,12 +56,13 @@ pipeline { stage('Run Daily Policies') { steps { script { - for (int i = 0; i < account.size(); ++i) { - echo "Running for account ${account[i].toUpperCase()}" - withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), - string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), - string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { - env.account_name = "${account[i]}" + for (account in accounts_list.keySet()) { + echo "Running for account ${account.toUpperCase()}" + withCredentials([string(credentialsId: "${account}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account}" + env.ADMIN_MAIL_LIST = "${accounts_list[account]}" sh 'python3 jenkins/tenant/aws/common/run_policies.py' } } diff --git a/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily index 61d80f81d..03510b1d5 100644 --- a/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily @@ -1,4 +1,6 @@ -account = ['appeng', 'fsi-partner', 'fsi-ecosystem'] +account = ['appeng': "mmikhail@redhat.com, matrujil@redhat.com, babak@redhat.com", + 'fsi-partner': "mmikhail@redhat.com, avillega@redhat.com, babak@redhat.com", + 'fsi-ecosystem': "mmikhail@redhat.com, avillega@redhat.com, babak@redhat.com"] pipeline { options { disableConcurrentBuilds() @@ -60,6 +62,7 @@ pipeline { string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { env.account_name = "${account[i]}" + env.ADMIN_MAIL_LIST = "${accounts_list[account]}" sh 'python3 jenkins/tenant/aws/common/run_policies.py' } } diff --git a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily index 706021e17..43134af3b 100644 --- a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily @@ -1,4 +1,11 @@ -account = ['coreos-training', 'industry-partners', 'special-projects', 'edgeinfra', 'specialprojects-qe', 'ecoeng-sap', 'sysdeseng', 'verticals-ui'] +account = ['coreos-training': "matt.dorn@redhat.com, babak@redhat.com", + 'industry-partners': "", + 'special-projects': "", + 'edgeinfra': "", + 'specialprojects-qe': "", + 'ecoeng-sap': "", + 'sysdeseng': "", + 'verticals-ui': ""] pipeline { options { disableConcurrentBuilds() @@ -60,6 +67,7 @@ pipeline { string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { env.account_name = "${account[i]}" + env.ADMIN_MAIL_LIST = "${accounts_list[account]}" sh 'python3 jenkins/tenant/aws/common/run_policies.py' } }