diff --git a/jenkins/poc/haim/appeng/PolicyJenkinsfileDaily b/jenkins/poc/haim/appeng/PolicyJenkinsfileDaily deleted file mode 100644 index 04de5201..00000000 --- a/jenkins/poc/haim/appeng/PolicyJenkinsfileDaily +++ /dev/null @@ -1,63 +0,0 @@ -pipeline { - options { - disableConcurrentBuilds() - } - agent { - docker { - label 'cloud-governance-worker' - image 'quay.io/athiru/centos-stream8-podman:latest' - args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' - } - } - environment { - access_key = credentials('appeng-aws-access-key-id') - secret_key = credentials('appeng-aws-secret-key-id') - s3_bucket = credentials('appeng-s3-bucket') - account_name = "appeng" - AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') - GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') - LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') - ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') - ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') - contact1 = "ebattat@redhat.com" - contact2 = "athiruma@redhat.com" - } - stages { - stage('Checkout') { // Checkout (git clone ...) the projects repository - steps { - checkout scm - } - } - stage('Initial Cleanup') { - steps { - sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' - } - } - stage('Run Policies the Cost Policies') { - steps { - sh 'python3 jenkins/poc/haim/common/run_cost_policies.py' - } - } - stage('Run Policies the Daily polices') { - steps { - sh 'python3 jenkins/poc/haim/common/run_policies.py' - } - } - stage('Finalize Cleanup') { - steps { - sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' - deleteDir() - } - } - } - post { - failure { - script { - msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" - emailext body: """\ - Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n - """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}" - } - } - } -} diff --git a/jenkins/poc/haim/appeng/TaggingJenkinsfileHourly b/jenkins/poc/haim/appeng/TaggingJenkinsfileHourly deleted file mode 100644 index aad0d7f7..00000000 --- a/jenkins/poc/haim/appeng/TaggingJenkinsfileHourly +++ /dev/null @@ -1,53 +0,0 @@ -pipeline { - options { - disableConcurrentBuilds() - } - agent { - docker { - label 'cloud-governance-worker' - image 'quay.io/athiru/centos-stream8-podman:latest' - args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' - } - } - environment { - access_key = credentials('appeng-aws-access-key-id') - secret_key = credentials('appeng-aws-secret-key-id') - LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') - account_name = "appeng" - contact1 = "ebattat@redhat.com" - contact2 = "athiruma@redhat.com" - } - stages { - stage('Checkout') { // Checkout (git clone ...) the projects repository - steps { - checkout scm - } - } - stage('Initial Cleanup') { - steps { - sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' - } - } - stage('Run Tagging Cluster & Non-Cluster') { - steps { - sh 'python3 jenkins/poc/haim/common/run_tagging.py' - } - } - stage('Finalize Cleanup') { - steps { - sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' - deleteDir() - } - } - } - post { - failure { - script { - msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" - emailext body: """\ - Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n - """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}" - } - } - } -} diff --git a/jenkins/poc/haim/common/run_cost_policies.py b/jenkins/poc/haim/common/run_cost_policies.py deleted file mode 100644 index 8fd826f6..00000000 --- a/jenkins/poc/haim/common/run_cost_policies.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -access_key = os.environ['access_key'] -secret_key = os.environ['secret_key'] -s3_bucket = os.environ['s3_bucket'] -account_name = os.environ['account_name'] -ES_HOST = os.environ['ES_HOST'] -ES_PORT = os.environ['ES_PORT'] - -cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', - 'LaunchTime', 'Name', 'Email', 'Environment', 'User:Spot'] -cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost -granularity = 'DAILY' # DAILY/MONTHLY/HOURLY -cost_explorer_index = 'cloud-governance-haim-cost-explorer-global-index' -os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """) -os.system( - f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/common/run_policies.py b/jenkins/poc/haim/common/run_policies.py deleted file mode 100644 index cc39b2d1..00000000 --- a/jenkins/poc/haim/common/run_policies.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -from ast import literal_eval - -policies_in_action = ['ebs_unattached', 'ip_unattached', 'zombie_snapshots', 'unused_nat_gateway', 's3_inactive', - 'empty_roles'] -policies_not_in_action = ['ec2_stop', 'instance_idle', 'zombie_cluster_resource'] - -access_key = os.environ['access_key'] -secret_key = os.environ['secret_key'] -s3_bucket = os.environ['s3_bucket'] -account_name = os.environ['account_name'] -days_to_delete_resource = os.environ.get('days_to_delete_resource', 14) -LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] -LOGS = os.environ.get('LOGS', 'logs') -ES_HOST = os.environ['ES_HOST'] -ES_PORT = os.environ['ES_PORT'] -GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] -SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] - -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] - -es_doc_type = '_doc' - -os.system(f"""echo Running the cloud_governance policies with dry_run=yes""") -os.system(f"echo Polices list: {policies_not_in_action}") -for region in regions: - for policy in policies_not_in_action: - os.system( - f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") - if policy == 'zombie_cluster_resource': - os.system( - f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="{account_name}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_doc_type="{es_doc_type}" -e bucket="{s3_bucket}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") - -os.system('echo "Running the CloudGovernance policies with dry_run=no" ') -os.system(f"echo Polices list: {policies_in_action}") -for region in regions: - for policy in policies_in_action: - if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1': - os.system( - f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") - elif policy not in ('empty_roles', 's3_inactive'): - os.system( - f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") - -os.system(f"""echo "Running the tag_iam_user" """) -os.system( - f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/common/run_tagging.py b/jenkins/poc/haim/common/run_tagging.py deleted file mode 100644 index d185d1ec..00000000 --- a/jenkins/poc/haim/common/run_tagging.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -access_key = os.environ['access_key'] -secret_key = os.environ['secret_key'] -account_name = os.environ['account_name'] -LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] - -LOGS = os.environ.get('LOGS', 'logs') - -mandatory_tags_appeng = {'Budget': account_name} - -os.system(f"""echo "Running the tag_resources" """) -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] -for region in regions: - os.system( - f"""podman run --rm --name cloud-governance-poc-haim -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/daily/Jenkinsfile b/jenkins/poc/haim/daily/Jenkinsfile deleted file mode 100644 index 52c458e4..00000000 --- a/jenkins/poc/haim/daily/Jenkinsfile +++ /dev/null @@ -1,62 +0,0 @@ -pipeline { - options { - disableConcurrentBuilds() - } - agent { - docker { - label 'cloud-governance-worker' - image 'quay.io/athiru/centos-stream8-podman:latest' - args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' - } - } - environment { - AWS_ACCESS_KEY_ID_APPENG = credentials('cloud-governance-aws-access-key-id-appeng') - AWS_SECRET_ACCESS_KEY_APPENG = credentials('cloud-governance-aws-secret-access-key-appeng') - AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') - GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') - LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') - ES_HOST = credentials('cloud-governance-es-host') - ES_PORT = credentials('cloud-governance-es-port') - BUCKET_APPENG = credentials('cloud-governance-bucket-appeng') - contact1 = "ebattat@redhat.com" - contact2 = "athiruma@redhat.com" - } - stages { - stage('Checkout') { // Checkout (git clone ...) the projects repository - steps { - checkout scm - } - } - stage('Initial Cleanup') { - steps { - sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' - } - } - stage('Run Policies for haim poc') { - steps { - sh 'python3 jenkins/poc/haim/daily/run_policies.py' - } - } - stage('Upload Policies output to ElasticSearch for haim poc') { - steps { - sh 'python3 jenkins/poc/haim/daily/es_upload.py' - } - } - stage('Finalize Cleanup') { - steps { - sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' - deleteDir() - } - } - } - post { - failure { - script { - msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" - emailext body: """\ - Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n - """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}" - } - } - } -} diff --git a/jenkins/poc/haim/daily/es_upload.py b/jenkins/poc/haim/daily/es_upload.py deleted file mode 100644 index 58bd0d22..00000000 --- a/jenkins/poc/haim/daily/es_upload.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -AWS_ACCESS_KEY_ID_APPENG = os.environ['AWS_ACCESS_KEY_ID_APPENG'] -AWS_SECRET_ACCESS_KEY_APPENG = os.environ['AWS_SECRET_ACCESS_KEY_APPENG'] -LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] -BUCKET_APPENG = os.environ['BUCKET_APPENG'] -ES_HOST = os.environ['ES_HOST'] -ES_PORT = os.environ['ES_PORT'] -LOGS = os.environ.get('LOGS', 'logs') - - -def get_policies(type: str = None): - """ - This method return a list of policies name without extension, that can filter by type - @return: list of policies name - """ - policies = [] - policies_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))), - 'cloud_governance', 'policy', 'aws') - for (dirpath, dirnames, filenames) in os.walk(policies_path): - for filename in filenames: - if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')): - if not type: - policies.append(os.path.splitext(filename)[0]) - elif type and type in filename: - policies.append(os.path.splitext(filename)[0]) - return policies - - -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] - -os.system('echo "Upload data to ElasticSearch - ec2 index"') - -es_index = 'cloud-governance-appeng-ec2-index' -es_doc_type = '_doc' -for region in regions: - for policy_types in ['ec2', 'zombie', 'ebs', 'empty_roles', 's3', 'ip', 'nat_gateway_unused']: - policies = get_policies(type=policy_types) - for policy in policies: - if policy in ('empty_roles', 's3_inactive'): - if region == 'us-east-1': - os.system( - f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="APPENG" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index}" -e es_doc_type="{es_doc_type}" -e bucket="{BUCKET_APPENG}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") - else: - os.system( - f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="APPENG" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index}" -e es_doc_type="{es_doc_type}" -e bucket="{BUCKET_APPENG}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/daily/run_policies.py b/jenkins/poc/haim/daily/run_policies.py deleted file mode 100644 index 2bc22c6a..00000000 --- a/jenkins/poc/haim/daily/run_policies.py +++ /dev/null @@ -1,57 +0,0 @@ -import os - -AWS_ACCESS_KEY_ID_APPENG = os.environ['AWS_ACCESS_KEY_ID_APPENG'] -AWS_SECRET_ACCESS_KEY_APPENG = os.environ['AWS_SECRET_ACCESS_KEY_APPENG'] -LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] -LOGS = os.environ.get('LOGS', 'logs') -ES_HOST = os.environ['ES_HOST'] -ES_PORT = os.environ['ES_PORT'] -BUCKET_APPENG = os.environ['BUCKET_APPENG'] - - -def get_policies(type: str = None): - """ - This method return a list of policies name without extension, that can filter by type - @return: list of policies name - """ - policies = [] - policies_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))), - 'cloud_governance', 'policy', 'aws') - for (dirpath, dirnames, filenames) in os.walk(policies_path): - for filename in filenames: - if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')): - if not type: - policies.append(os.path.splitext(filename)[0]) - elif type and type in filename: - policies.append(os.path.splitext(filename)[0]) - return policies - - -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] -policies = get_policies() -not_action_policies = ['cost_explorer', 'cost_over_usage', 'monthly_report', 'cost_billing_reports', - 'cost_explorer_payer_billings'] -run_policies = list(set(policies) - set(not_action_policies)) -run_policies.sort() - -os.system(f"""echo Running the cloud_governance policies: {run_policies}""") -os.system(f"""echo "Running the CloudGovernance policies" """) -for region in regions: - for policy in run_policies: - if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1': - os.system( - f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="APPENG" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{BUCKET_APPENG}/{LOGS}/{region}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") - else: - os.system( - f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="APPENG" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{BUCKET_APPENG}/{LOGS}/{region}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") - -cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', - 'Email', 'Environment', 'User:Spot'] -cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost -granularity = 'DAILY' # DAILY/MONTHLY/HOURLY -cost_explorer_index = 'cloud-governance-haim-cost-explorer-global-index' -os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """) -os.system( - f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="appeng" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/hourly/Jenkinsfile b/jenkins/poc/haim/hourly/Jenkinsfile deleted file mode 100644 index 0715995b..00000000 --- a/jenkins/poc/haim/hourly/Jenkinsfile +++ /dev/null @@ -1,54 +0,0 @@ -pipeline { - options { - disableConcurrentBuilds() - } - agent { - docker { - label 'cloud-governance-worker' - image 'quay.io/athiru/centos-stream8-podman:latest' - args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' - } - } - environment { - AWS_ACCESS_KEY_ID_APPENG = credentials('cloud-governance-aws-access-key-id-appeng') - AWS_SECRET_ACCESS_KEY_APPENG = credentials('cloud-governance-aws-secret-access-key-appeng') - AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') - GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') - LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') - contact1 = "ebattat@redhat.com" - contact2 = "athiruma@redhat.com" - } - stages { - stage('Checkout') { // Checkout (git clone ...) the projects repository - steps { - checkout scm - } - } - stage('Initial Cleanup') { - steps { - sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' - } - } - stage('Run Tagging Cluster & Non-Cluster') { - steps { - sh 'python3 jenkins/poc/haim/hourly/run_policies.py' - } - } - stage('Finalize Cleanup') { - steps { - sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' - deleteDir() - } - } - } - post { - failure { - script { - msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" - emailext body: """\ - Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n - """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}" - } - } - } -} diff --git a/jenkins/poc/haim/hourly/run_policies.py b/jenkins/poc/haim/hourly/run_policies.py deleted file mode 100644 index 9fdbf21e..00000000 --- a/jenkins/poc/haim/hourly/run_policies.py +++ /dev/null @@ -1,22 +0,0 @@ -import os - -AWS_ACCESS_KEY_ID_APPENG = os.environ['AWS_ACCESS_KEY_ID_APPENG'] -AWS_SECRET_ACCESS_KEY_APPENG = os.environ['AWS_SECRET_ACCESS_KEY_APPENG'] -LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] -GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] -SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] - -LOGS = os.environ.get('LOGS', 'logs') - -mandatory_tags_appeng = {'Budget': 'APPENG'} - -os.system(f"""echo "Running the tag_iam_user" """) -os.system( - f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="APPENG" -e -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") - -os.system(f"""echo "Running the tag_resources" """) -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] -for region in regions: - os.system( - f"""podman run --rm --name cloud-governance-poc-haim -e account="APPENG" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/tenant/aws/common/run_policies.py b/jenkins/tenant/aws/common/run_policies.py index 6f79ed98..93db34a0 100644 --- a/jenkins/tenant/aws/common/run_policies.py +++ b/jenkins/tenant/aws/common/run_policies.py @@ -126,4 +126,4 @@ def run_policies(policies: list, dry_run: str = 'yes'): # Run the AggMail run_cmd( - f"""podman run --rm --name cloud-governance-haim --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" -e ALERT_DRY_RUN="{ALERT_DRY_RUN}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --name cloud-governance-haim --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {env_es_index} -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" -e ALERT_DRY_RUN="{ALERT_DRY_RUN}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""")