Skip to content

Commit

Permalink
Added the tenants: ecoeng_01, ecoeng_02, ecoeng_03 (#672)
Browse files Browse the repository at this point in the history
* Added the tenanats: ecoeng, mikhail

* Added the new acconuts

* Added the policy to the Jenkinsfile
  • Loading branch information
athiruma authored Jan 10, 2024
1 parent 28ebbdb commit 87b2f06
Show file tree
Hide file tree
Showing 11 changed files with 559 additions and 20 deletions.
20 changes: 20 additions & 0 deletions cloud_governance/common/tool/tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,23 @@ def is_empty_file(file_path):
"""
if os.stat(file_path).st_size == 0:
raise Exception(f'File is empty: {file_path}')


def get_policies(file_type: str = '.py', exclude_policies: list = None):
"""
This method return a list of policies name without extension, that can filter by type
@return: list of custodian policies name
"""
exclude_policies = [] if not exclude_policies else exclude_policies
custodian_policies = []
root_folder = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
policies_path = os.path.join(root_folder, 'policy', 'aws')
for (_, _, filenames) in os.walk(policies_path):
for filename in filenames:
if not filename.startswith('__') and filename.endswith(file_type):
if filename.split('.')[0] not in exclude_policies:
if not file_type:
custodian_policies.append(os.path.splitext(filename)[0])
elif file_type and file_type in filename:
custodian_policies.append(os.path.splitext(filename)[0])
return custodian_policies
88 changes: 68 additions & 20 deletions jenkins/tenant/aws/common/run_policies.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,34 @@

import os
from ast import literal_eval

policies_in_action = []
policies_not_in_action = ['ec2_stop', 'ec2_idle', 'zombie_cluster_resource', 'ebs_unattached', 'ip_unattached',
'zombie_snapshots', 'unused_nat_gateway', 's3_inactive', 'empty_roles']
from cloud_governance.common.tool.tool import get_policies

exclude_policies = ['cost_explorer', 'optimize_resources_report', 'monthly_report', 'cost_over_usage',
'skipped_resources', 'cost_explorer_payer_billings', 'cost_billing_reports', 'spot_savings_analysis']
available_policies = get_policies(exclude_policies=exclude_policies)
# available_policies: Run policies in dry_run="yes" mode


def run_cmd(cmd: str):
"""
This method run the command
:param cmd:
:type cmd:
:return:
:rtype:
"""
print(cmd)


def get_container_cmd(env_dict: dict):
create_container_envs = lambda item: f'-e {item[0]}="{item[1]}"'
env_list = ' '.join(list(map(create_container_envs, env_dict.items())))
container_name = "cloud-governance-poc-haim"
container_run_cmd = f"""
podman run --rm --name "{container_name}" --net="host" {env_list} quay.io/ebattat/cloud-governance:latest
"""
return container_run_cmd


access_key = os.environ['access_key']
secret_key = os.environ['secret_key']
Expand All @@ -18,27 +42,51 @@
GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS']
SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID']

regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1']
policies_in_action = os.environ.get('POLICIES_IN_ACTION', [])
if isinstance(policies_in_action, str):
policies_in_action = literal_eval(policies_in_action)
policies_not_action = list(set(available_policies) - set(policies_in_action))


regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2',
'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1',
'ap-southeast-1', 'ap-southeast-2', 'eu-central-1']

es_doc_type = '_doc'

os.system(f"""echo Running the cloud_governance policies with dry_run=yes""")
os.system(f"echo Polices list: {policies_not_in_action}")
for region in regions:
for policy in policies_not_in_action:
os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
container_env_dict = {
"account": account_name, "AWS_DEFAULT_REGION": "us-east-1", "PUBLIC_CLOUD_NAME": "AWS",
"AWS_ACCESS_KEY_ID": access_key, "AWS_SECRET_ACCESS_KEY": secret_key,
"dry_run": "yes", "LDAP_HOST_NAME": LDAP_HOST_NAME, "DAYS_TO_DELETE_RESOURCE": days_to_delete_resource,
"es_host": ES_HOST, "es_port": ES_PORT,
"MANAGER_EMAIL_ALERT": "False", "EMAIL_ALERT": "False", "log_level": "INFO"
}


def run_policies(policies: list, dry_run: str = 'yes'):
for region in regions:
container_env_dict.update({"policy_output": f"s3://{s3_bucket}/{LOGS}/{region}", "AWS_DEFAULT_REGION": region,
'dry_run': dry_run})
for policy in policies:
container_env_dict.update({"AWS_DEFAULT_REGION": region, 'policy': policy})
container_cmd = ''
if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1':
container_cmd = get_container_cmd(container_env_dict)
else:
if policy not in ('empty_roles', 's3_inactive'):
container_cmd = get_container_cmd(container_env_dict)
if container_cmd:
run_cmd(container_cmd)


run_cmd(f"echo Running the cloud_governance policies with dry_run=yes")
run_cmd(f"echo Polices list: {policies_not_action}")
run_policies(policies=policies_not_action)

os.system('echo "Running the CloudGovernance policies with dry_run=no" ')
os.system(f"echo Polices list: {policies_in_action}")
for region in regions:
for policy in policies_in_action:
if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1':
os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
elif policy not in ('empty_roles', 's3_inactive'):
os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
run_cmd('echo "Running the CloudGovernance policies with dry_run=no" ')
run_cmd(f"echo Polices list: {policies_in_action}")
run_policies(policies=policies_in_action, dry_run='no')


os.system(f"""echo "Running the tag_iam_user" """)
os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
run_cmd(f"""echo "Running the tag_iam_user" """)
run_cmd(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
86 changes: 86 additions & 0 deletions jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
account = ['partnerlab', 'coreos-training']
pipeline {
options {
disableConcurrentBuilds()
}
agent {
docker {
label 'haim-cloud-governance-worker'
image 'quay.io/athiru/centos-stream8-podman:latest'
args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged'
}
}
environment {
AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id')
GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials')
LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name')
ES_HOST = credentials('haim-cloud-governance-elasticsearch-url')
ES_PORT = credentials('haim-cloud-governance-elasticsearch-port')
contact1 = "[email protected]"
contact2 = "[email protected]"
contact3 = "[email protected]"
contact4 = "[email protected]"
// Find the all available policies: https://github.com/redhat-performance/cloud-governance/tree/main/cloud_governance/policy
// By default, all policies are running in dry_run="yes" mode and the whole list can be found in run_policies.py
// POLICIES_IN_ACTION: Policies that run in the dry_run="no" mode
POLICIES_IN_ACTION = ["ebs_unattached", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles"]
}
stages {
stage('Checkout') { // Checkout (git clone ...) the projects repository
steps {
checkout scm
}
}
stage('Initial Cleanup') {
steps {
sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
}
}
stage('Run Cost Policies') {
steps {
script {
for (int i = 0; i < account.size(); ++i) {
echo "Running for account ${account[i].toUpperCase()}"
withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'),
string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'),
string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) {
env.account_name = "${account[i]}"
sh 'python3 jenkins/tenant/aws/common/run_cost_policies.py'
}
}
}
}
}
stage('Run Daily Policies') {
steps {
script {
for (int i = 0; i < account.size(); ++i) {
echo "Running for account ${account[i].toUpperCase()}"
withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'),
string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'),
string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) {
env.account_name = "${account[i]}"
sh 'python3 jenkins/tenant/aws/common/run_policies.py'
}
}
}
}
}
stage('Finalize Cleanup') {
steps {
sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
deleteDir()
}
}
}
post {
failure {
script {
msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})"
emailext body: """\
Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n
""",subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}"
}
}
}
}
5 changes: 5 additions & 0 deletions jenkins/tenant/aws/ecoeng_01/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
### ecoeng_01

Accounts:
- partnerlab
- coreos-training
64 changes: 64 additions & 0 deletions jenkins/tenant/aws/ecoeng_01/TaggingJenkinsfileHourly
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
account = ['partnerlab', 'coreos-training']
pipeline {
options {
disableConcurrentBuilds()
}
agent {
docker {
label 'haim-cloud-governance-worker'
image 'quay.io/athiru/centos-stream8-podman:latest'
args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged'
}
}
environment {
LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name')
account_name = "appeng"
contact1 = "[email protected]"
contact2 = "[email protected]"
contact3 = "[email protected]"
contact4 = "[email protected]"
}
stages {
stage('Checkout') { // Checkout (git clone ...) the projects repository
steps {
checkout scm
}
}
stage('Initial Cleanup') {
steps {
sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
}
}
stage('Run Tagging Cluster & Non-Cluster') {
steps {
script {
for (int i = 0; i < account.size(); ++i) {
echo "Running for account ${account[i].toUpperCase()}"
withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'),
string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'),
string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) {
env.account_name = "${account[i]}"
sh 'python3 jenkins/tenant/aws/common/run_tagging.py'
}
}
}
}
}
stage('Finalize Cleanup') {
steps {
sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
deleteDir()
}
}
}
post {
failure {
script {
msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})"
emailext body: """\
Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n
""",subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}"
}
}
}
}
86 changes: 86 additions & 0 deletions jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
account = ['appeng', 'fsi-partner', 'fsi-ecosystem']
pipeline {
options {
disableConcurrentBuilds()
}
agent {
docker {
label 'haim-cloud-governance-worker'
image 'quay.io/athiru/centos-stream8-podman:latest'
args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged'
}
}
environment {
AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id')
GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials')
LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name')
ES_HOST = credentials('haim-cloud-governance-elasticsearch-url')
ES_PORT = credentials('haim-cloud-governance-elasticsearch-port')
contact1 = "[email protected]"
contact2 = "[email protected]"
contact3 = "[email protected]"
contact4 = "[email protected]"
// Find the all available policies: https://github.com/redhat-performance/cloud-governance/tree/main/cloud_governance/policy
// By default, all policies are running in dry_run="yes" mode and the whole list can be found in run_policies.py
// POLICIES_IN_ACTION: Policies that run in the dry_run="no" mode
POLICIES_IN_ACTION = ["ebs_unattached", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles"]
}
stages {
stage('Checkout') { // Checkout (git clone ...) the projects repository
steps {
checkout scm
}
}
stage('Initial Cleanup') {
steps {
sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
}
}
stage('Run Cost Policies') {
steps {
script {
for (int i = 0; i < account.size(); ++i) {
echo "Running for account ${account[i].toUpperCase()}"
withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'),
string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'),
string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) {
env.account_name = "${account[i]}"
sh 'python3 jenkins/tenant/aws/common/run_cost_policies.py'
}
}
}
}
}
stage('Run Daily Policies') {
steps {
script {
for (int i = 0; i < account.size(); ++i) {
echo "Running for account ${account[i].toUpperCase()}"
withCredentials([string(credentialsId: "${account[i]}-aws-access-key-id", variable: 'access_key'),
string(credentialsId: "${account[i]}-aws-secret-key-id", variable: 'secret_key'),
string(credentialsId: "${account[i]}-s3-bucket", variable: 's3_bucket')]) {
env.account_name = "${account[i]}"
sh 'python3 jenkins/tenant/aws/common/run_policies.py'
}
}
}
}
}
stage('Finalize Cleanup') {
steps {
sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
deleteDir()
}
}
}
post {
failure {
script {
msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})"
emailext body: """\
Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n
""",subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}"
}
}
}
}
6 changes: 6 additions & 0 deletions jenkins/tenant/aws/ecoeng_02/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
### ecoeng_02

Accounts:
- appeng
- fsi-partner
- fsi-ecosystem
Loading

0 comments on commit 87b2f06

Please sign in to comment.