diff --git a/jenkins/poc/haim/common/run_policies.py b/jenkins/poc/haim/common/run_policies.py index 20c71bdb..a3f3f594 100644 --- a/jenkins/poc/haim/common/run_policies.py +++ b/jenkins/poc/haim/common/run_policies.py @@ -40,4 +40,4 @@ os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") os.system(f"""echo "Running the tag_iam_user" """) -os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") diff --git a/jenkins/tenant/aws/README.md b/jenkins/tenant/aws/README.md new file mode 100644 index 00000000..afcae3a8 --- /dev/null +++ b/jenkins/tenant/aws/README.md @@ -0,0 +1,12 @@ +# How to run cloud-governance on Tenant Accounts + +Steps +1. Create AWS User and attach user by [CloudGovernanceDeletePolicy.json](../../../iam/clouds/aws/CloudGovernanceDeletePolicy.json). [ Note: Replace account_id with actual account id] +2. Create S3 bucket +3. Add kind secret-text to jenkins with below naming conventions + 1. ${account_name}-aws-access-key-id + 2. ${account_name}-aws-secret-key-id + 3. ${account_name}-s3-bucket +4. Create folder named that you want to run the cloud-governance policies and copy the file in templates. +5. Add account_name to account variable in this [PolicyJenkinsfileDaily](../aws/template/PolicyJenkinsfileDaily) and [TaggingJenkinsfileHourly](../aws/template/TaggingJenkinsfileHourly). +6. Create two Jenkins jobs by using this two Jenkinsfile diff --git a/jenkins/tenant/aws/common/run_cost_policies.py b/jenkins/tenant/aws/common/run_cost_policies.py new file mode 100644 index 00000000..a8e4440f --- /dev/null +++ b/jenkins/tenant/aws/common/run_cost_policies.py @@ -0,0 +1,17 @@ +import os + +access_key = os.environ['access_key'] +secret_key = os.environ['secret_key'] +s3_bucket = os.environ['s3_bucket'] +account_name = os.environ['account_name'] +ES_HOST = os.environ['ES_HOST'] +ES_PORT = os.environ['ES_PORT'] + + +cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', + 'LaunchTime', 'Name', 'Email', 'Environment', 'User:Spot'] +cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost +granularity = 'DAILY' # DAILY/MONTHLY/HOURLY +cost_explorer_index = 'cloud-governance-haim-cost-explorer-global-index' +os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """) +os.system(f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") diff --git a/jenkins/tenant/aws/common/run_policies.py b/jenkins/tenant/aws/common/run_policies.py new file mode 100644 index 00000000..764ecc62 --- /dev/null +++ b/jenkins/tenant/aws/common/run_policies.py @@ -0,0 +1,47 @@ + +import os +from ast import literal_eval + +policies_in_action = [] +policies_not_in_action = ['ec2_stop', 'ec2_idle', 'zombie_cluster_resource', 'ebs_unattached', 'ip_unattached', + 'zombie_snapshots', 'unused_nat_gateway', 's3_inactive', 'empty_roles'] + +access_key = os.environ['access_key'] +secret_key = os.environ['secret_key'] +s3_bucket = os.environ['s3_bucket'] +account_name = os.environ['account_name'] +days_to_delete_resource = os.environ.get('days_to_delete_resource', 14) +LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] +LOGS = os.environ.get('LOGS', 'logs') +ES_HOST = os.environ['ES_HOST'] +ES_PORT = os.environ['ES_PORT'] +GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] +SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] + +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', + 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] + +es_doc_type = '_doc' + +os.system(f"""echo Running the cloud_governance policies with dry_run=yes""") +os.system(f"echo Polices list: {policies_not_in_action}") +for region in regions: + for policy in policies_not_in_action: + os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + if policy == 'zombie_cluster_resource': + os.system(f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="{account_name}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_doc_type="{es_doc_type}" -e bucket="{s3_bucket}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + +os.system('echo "Running the CloudGovernance policies with dry_run=no" ') +os.system(f"echo Polices list: {policies_in_action}") +for region in regions: + for policy in policies_in_action: + if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1': + os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + elif policy not in ('empty_roles', 's3_inactive'): + os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + if policy == 'zombie_cluster_resource': + os.system(f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="{account_name}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_doc_type="{es_doc_type}" -e bucket="{s3_bucket}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + + +os.system(f"""echo "Running the tag_iam_user" """) +os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") diff --git a/jenkins/tenant/aws/common/run_tagging.py b/jenkins/tenant/aws/common/run_tagging.py new file mode 100644 index 00000000..8cfb017e --- /dev/null +++ b/jenkins/tenant/aws/common/run_tagging.py @@ -0,0 +1,19 @@ + +import os + + +access_key = os.environ['access_key'] +secret_key = os.environ['secret_key'] +account_name = os.environ['account_name'] +LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] + + +LOGS = os.environ.get('LOGS', 'logs') + +mandatory_tags_appeng = {'Budget': account_name} + + +os.system(f"""echo "Running the tag_resources" """) +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] +for region in regions: + os.system(f"""podman run --rm --name cloud-governance-poc-haim -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") diff --git a/jenkins/tenant/aws/default/PolicyJenkinsfileDaily b/jenkins/tenant/aws/default/PolicyJenkinsfileDaily new file mode 100644 index 00000000..08c94606 --- /dev/null +++ b/jenkins/tenant/aws/default/PolicyJenkinsfileDaily @@ -0,0 +1,80 @@ +account = [] +pipeline { + options { + disableConcurrentBuilds() + } + agent { + docker { + label 'cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') + GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') + ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') + contact1 = "ebattat@redhat.com" + contact2 = "athiruma@redhat.com" + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Policies the Cost Policies') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/poc/haim/common/run_cost_policies.py' + } + } + } + } + } + stage('Run Policies the Daily polices') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/poc/haim/common/run_policies.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}" + } + } + } +} diff --git a/jenkins/tenant/aws/default/TaggingJenkinsfileHourly b/jenkins/tenant/aws/default/TaggingJenkinsfileHourly new file mode 100644 index 00000000..1cdae307 --- /dev/null +++ b/jenkins/tenant/aws/default/TaggingJenkinsfileHourly @@ -0,0 +1,62 @@ +account = [] +pipeline { + options { + disableConcurrentBuilds() + } + agent { + docker { + label 'cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + account_name = "appeng" + contact1 = "ebattat@redhat.com" + contact2 = "athiruma@redhat.com" + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Tagging Cluster & Non-Cluster') { + steps { + script { + for (int i = 0; i < account.size(); ++i) { + echo "Running for account ${account[i].toUpperCase()}" + withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account[i]}" + sh 'python3 jenkins/poc/haim/common/run_tagging.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}" + } + } + } +}