diff --git a/cloud_governance/common/tool/tool.py b/cloud_governance/common/tool/tool.py index 92492412..2dc11b7b 100644 --- a/cloud_governance/common/tool/tool.py +++ b/cloud_governance/common/tool/tool.py @@ -9,3 +9,23 @@ def is_empty_file(file_path): """ if os.stat(file_path).st_size == 0: raise Exception(f'File is empty: {file_path}') + + +def get_policies(file_type: str = '.py', exclude_policies: list = None): + """ + This method return a list of policies name without extension, that can filter by type + @return: list of custodian policies name + """ + exclude_policies = [] if not exclude_policies else exclude_policies + custodian_policies = [] + root_folder = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + policies_path = os.path.join(root_folder, 'policy', 'aws') + for (_, _, filenames) in os.walk(policies_path): + for filename in filenames: + if not filename.startswith('__') and filename.endswith(file_type): + if filename.split('.')[0] not in exclude_policies: + if not file_type: + custodian_policies.append(os.path.splitext(filename)[0]) + elif file_type and file_type in filename: + custodian_policies.append(os.path.splitext(filename)[0]) + return custodian_policies diff --git a/jenkins/tenant/aws/common/run_policies.py b/jenkins/tenant/aws/common/run_policies.py index b767c774..da67f959 100644 --- a/jenkins/tenant/aws/common/run_policies.py +++ b/jenkins/tenant/aws/common/run_policies.py @@ -1,10 +1,34 @@ - import os from ast import literal_eval -policies_in_action = [] -policies_not_in_action = ['ec2_stop', 'ec2_idle', 'zombie_cluster_resource', 'ebs_unattached', 'ip_unattached', - 'zombie_snapshots', 'unused_nat_gateway', 's3_inactive', 'empty_roles'] +from cloud_governance.common.tool.tool import get_policies + +exclude_policies = ['cost_explorer', 'optimize_resources_report', 'monthly_report', 'cost_over_usage', + 'skipped_resources', 'cost_explorer_payer_billings', 'cost_billing_reports', 'spot_savings_analysis'] +available_policies = get_policies(exclude_policies=exclude_policies) +# available_policies: Run policies in dry_run="yes" mode + + +def run_cmd(cmd: str): + """ + This method run the command + :param cmd: + :type cmd: + :return: + :rtype: + """ + print(cmd) + + +def get_container_cmd(env_dict: dict): + create_container_envs = lambda item: f'-e {item[0]}="{item[1]}"' + env_list = ' '.join(list(map(create_container_envs, env_dict.items()))) + container_name = "cloud-governance-poc-haim" + container_run_cmd = f""" +podman run --rm --name "{container_name}" --net="host" {env_list} quay.io/ebattat/cloud-governance:latest +""" + return container_run_cmd + access_key = os.environ['access_key'] secret_key = os.environ['secret_key'] @@ -18,27 +42,51 @@ GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1'] +policies_in_action = os.environ.get('POLICIES_IN_ACTION', []) +if isinstance(policies_in_action, str): + policies_in_action = literal_eval(policies_in_action) +policies_not_action = list(set(available_policies) - set(policies_in_action)) + +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', + 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1', + 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1'] es_doc_type = '_doc' -os.system(f"""echo Running the cloud_governance policies with dry_run=yes""") -os.system(f"echo Polices list: {policies_not_in_action}") -for region in regions: - for policy in policies_not_in_action: - os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +container_env_dict = { + "account": account_name, "AWS_DEFAULT_REGION": "us-east-1", "PUBLIC_CLOUD_NAME": "AWS", + "AWS_ACCESS_KEY_ID": access_key, "AWS_SECRET_ACCESS_KEY": secret_key, + "dry_run": "yes", "LDAP_HOST_NAME": LDAP_HOST_NAME, "DAYS_TO_DELETE_RESOURCE": days_to_delete_resource, + "es_host": ES_HOST, "es_port": ES_PORT, + "MANAGER_EMAIL_ALERT": "False", "EMAIL_ALERT": "False", "log_level": "INFO" +} + + +def run_policies(policies: list, dry_run: str = 'yes'): + for region in regions: + container_env_dict.update({"policy_output": f"s3://{s3_bucket}/{LOGS}/{region}", "AWS_DEFAULT_REGION": region, + 'dry_run': dry_run}) + for policy in policies: + container_env_dict.update({"AWS_DEFAULT_REGION": region, 'policy': policy}) + container_cmd = '' + if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1': + container_cmd = get_container_cmd(container_env_dict) + else: + if policy not in ('empty_roles', 's3_inactive'): + container_cmd = get_container_cmd(container_env_dict) + if container_cmd: + run_cmd(container_cmd) + +run_cmd(f"echo Running the cloud_governance policies with dry_run=yes") +run_cmd(f"echo Polices list: {policies_not_action}") +run_policies(policies=policies_not_action) -os.system('echo "Running the CloudGovernance policies with dry_run=no" ') -os.system(f"echo Polices list: {policies_in_action}") -for region in regions: - for policy in policies_in_action: - if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1': - os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") - elif policy not in ('empty_roles', 's3_inactive'): - os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +run_cmd('echo "Running the CloudGovernance policies with dry_run=no" ') +run_cmd(f"echo Polices list: {policies_in_action}") +run_policies(policies=policies_in_action, dry_run='no') -os.system(f"""echo "Running the tag_iam_user" """) -os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +run_cmd(f"""echo "Running the tag_iam_user" """) +run_cmd(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") diff --git a/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily new file mode 100644 index 00000000..cefcd932 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily @@ -0,0 +1,86 @@ +account = ['partnerlab', 'coreos-training'] +pipeline { + options { + disableConcurrentBuilds() + } + agent { + docker { + label 'haim-cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') + GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') + ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') + contact1 = "ebattat@redhat.com" + contact2 = "athiruma@redhat.com" + contact3 = "mmikhail@redhat.com" + contact4 = "matrujil@redhat.com" + // Find the all available policies: https://github.com/redhat-performance/cloud-governance/tree/main/cloud_governance/policy + // By default, all policies are running in dry_run="yes" mode and the whole list can be found in run_policies.py + // POLICIES_IN_ACTION: Policies that run in the dry_run="no" mode + POLICIES_IN_ACTION = ["ebs_unattached", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles"] + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Cost Policies') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_cost_policies.py' + } + } + } + } + } + stage('Run Daily Policies') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_policies.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}" + } + } + } +} diff --git a/jenkins/tenant/aws/ecoeng_01/README.md b/jenkins/tenant/aws/ecoeng_01/README.md new file mode 100644 index 00000000..3dfc0814 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_01/README.md @@ -0,0 +1,5 @@ +### ecoeng_01 + +Accounts: +- partnerlab +- coreos-training \ No newline at end of file diff --git a/jenkins/tenant/aws/ecoeng_01/TaggingJenkinsfileHourly b/jenkins/tenant/aws/ecoeng_01/TaggingJenkinsfileHourly new file mode 100644 index 00000000..c58204c4 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_01/TaggingJenkinsfileHourly @@ -0,0 +1,64 @@ +account = ['partnerlab', 'coreos-training'] +pipeline { + options { + disableConcurrentBuilds() + } + agent { + docker { + label 'haim-cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + account_name = "appeng" + contact1 = "ebattat@redhat.com" + contact2 = "athiruma@redhat.com" + contact3 = "mmikhail@redhat.com" + contact4 = "matrujil@redhat.com" + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Tagging Cluster & Non-Cluster') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_tagging.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}" + } + } + } +} diff --git a/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily new file mode 100644 index 00000000..eb564065 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily @@ -0,0 +1,86 @@ +account = ['appeng', 'fsi-partner', 'fsi-ecosystem'] +pipeline { + options { + disableConcurrentBuilds() + } + agent { + docker { + label 'haim-cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') + GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') + ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') + contact1 = "ebattat@redhat.com" + contact2 = "athiruma@redhat.com" + contact3 = "mmikhail@redhat.com" + contact4 = "matrujil@redhat.com" + // Find the all available policies: https://github.com/redhat-performance/cloud-governance/tree/main/cloud_governance/policy + // By default, all policies are running in dry_run="yes" mode and the whole list can be found in run_policies.py + // POLICIES_IN_ACTION: Policies that run in the dry_run="no" mode + POLICIES_IN_ACTION = ["ebs_unattached", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles"] + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Cost Policies') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_cost_policies.py' + } + } + } + } + } + stage('Run Daily Policies') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_policies.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}" + } + } + } +} diff --git a/jenkins/tenant/aws/ecoeng_02/README.md b/jenkins/tenant/aws/ecoeng_02/README.md new file mode 100644 index 00000000..ede98368 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_02/README.md @@ -0,0 +1,6 @@ +### ecoeng_02 + +Accounts: +- appeng +- fsi-partner +- fsi-ecosystem diff --git a/jenkins/tenant/aws/ecoeng_02/TaggingJenkinsfileHourly b/jenkins/tenant/aws/ecoeng_02/TaggingJenkinsfileHourly new file mode 100644 index 00000000..0a2b46d5 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_02/TaggingJenkinsfileHourly @@ -0,0 +1,64 @@ +account = ['appeng', 'fsi-partner', 'fsi-ecosystem'] +pipeline { + options { + disableConcurrentBuilds() + } + agent { + docker { + label 'haim-cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + account_name = "appeng" + contact1 = "ebattat@redhat.com" + contact2 = "athiruma@redhat.com" + contact3 = "mmikhail@redhat.com" + contact4 = "matrujil@redhat.com" + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Tagging Cluster & Non-Cluster') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_tagging.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}" + } + } + } +} diff --git a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily new file mode 100644 index 00000000..65b07849 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily @@ -0,0 +1,86 @@ +account = ['industry-partners', 'special-projects', 'edgeinfra', 'specialprojects-qe', 'ecoeng-sap', 'sysdeseng', 'verticals-ui'] +pipeline { + options { + disableConcurrentBuilds() + } + agent { + docker { + label 'haim-cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') + GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') + ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') + contact1 = "ebattat@redhat.com" + contact2 = "athiruma@redhat.com" + contact3 = "mmikhail@redhat.com" + contact4 = "matrujil@redhat.com" + // Find the all available policies: https://github.com/redhat-performance/cloud-governance/tree/main/cloud_governance/policy + // By default, all policies are running in dry_run="yes" mode and the whole list can be found in run_policies.py + // POLICIES_IN_ACTION: Policies that run in the dry_run="no" mode + POLICIES_IN_ACTION = [] + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Cost Policies') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_cost_policies.py' + } + } + } + } + } + stage('Run Daily Policies') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_policies.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}" + } + } + } +} diff --git a/jenkins/tenant/aws/ecoeng_03/README.md b/jenkins/tenant/aws/ecoeng_03/README.md new file mode 100644 index 00000000..8fd2c1f4 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_03/README.md @@ -0,0 +1,10 @@ +### ecoeng_03 + +Accounts: +- industry-partners +- special-projects +- edgeinfra +- specialprojects-qe +- ecoeng-sap +- sysdeseng +- verticals-ui \ No newline at end of file diff --git a/jenkins/tenant/aws/ecoeng_03/TaggingJenkinsfileHourly b/jenkins/tenant/aws/ecoeng_03/TaggingJenkinsfileHourly new file mode 100644 index 00000000..ec105366 --- /dev/null +++ b/jenkins/tenant/aws/ecoeng_03/TaggingJenkinsfileHourly @@ -0,0 +1,64 @@ +account = ['industry-partners', 'special-projects', 'edgeinfra', 'specialprojects-qe', 'ecoeng-sap', 'sysdeseng', 'verticals-ui'] +pipeline { + options { + disableConcurrentBuilds() + } + agent { + docker { + label 'haim-cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + account_name = "appeng" + contact1 = "ebattat@redhat.com" + contact2 = "athiruma@redhat.com" + contact3 = "mmikhail@redhat.com" + contact4 = "matrujil@redhat.com" + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Tagging Cluster & Non-Cluster') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/tenant/aws/common/run_tagging.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}" + } + } + } +}