From 260817b1a5df6fa66dd55afa81b9e8fa88460591 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Mon, 26 Feb 2024 15:04:47 +0530 Subject: [PATCH] Azure Policy: Jenkinsfile for running azure policies --- .../aws/daily/policies/azure_policies.py | 2 + .../clouds/azure/daily/policies/Jenkinsfile | 71 ++++++++++++ .../azure/daily/policies/run_policies.py | 105 ++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 jenkins/clouds/aws/daily/policies/azure_policies.py create mode 100644 jenkins/clouds/azure/daily/policies/Jenkinsfile create mode 100644 jenkins/clouds/azure/daily/policies/run_policies.py diff --git a/jenkins/clouds/aws/daily/policies/azure_policies.py b/jenkins/clouds/aws/daily/policies/azure_policies.py new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/jenkins/clouds/aws/daily/policies/azure_policies.py @@ -0,0 +1,2 @@ + + diff --git a/jenkins/clouds/azure/daily/policies/Jenkinsfile b/jenkins/clouds/azure/daily/policies/Jenkinsfile new file mode 100644 index 00000000..75e19438 --- /dev/null +++ b/jenkins/clouds/azure/daily/policies/Jenkinsfile @@ -0,0 +1,71 @@ +def accounts_list = ['perfscale'] +pipeline { + agent { + docker { + label 'cloud-governance-worker' + image 'quay.io/athiru/centos-stream8-podman:latest' + args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' + } + } + environment { + POLICIES_IN_ACTION = '[]' + AZURE_CLIENT_SECRET = credentials('cloud-governance-azure-client-secret') + AZURE_TENANT_ID = credentials('cloud-governance-azure-tenant-id') + AZURE_ACCOUNT_ID = credentials('cloud-governance-azure-account-id') + AZURE_CLIENT_ID = credentials('cloud-governance-azure-client-id') + ES_HOST = credentials('cloud-governance-es-host') + ES_PORT = credentials('cloud-governance-es-port') + LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + contact2 = "athiruma@redhat.com" + } + stages { + stage('Checkout') { // Checkout (git clone ...) the projects repository + steps { + checkout scm + } + } + stage('Initial Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + } + } + stage('Run Azure Policies') { + steps { + script { + for (account in accounts_list ) { + echo "Running for account ${account.toUpperCase()}" + withCredentials([string(credentialsId: "${account}-azure-client-secret", variable: 'client_secret'), + string(credentialsId: "${account}-azure-client-id", variable: 'client_id'), + string(credentialsId: "${account}-azure-tenant-id", variable: 'tenant_id'), + string(credentialsId: "${account}-azure-subscription-id", variable: 'subscription_id'), + string(credentialsId: "${account}-azure-account-id", variable: 'account_id')]) { + env.account_name = "Azure-${account}" + sh 'python3 jenkins/clouds/azure/daily/policies/run_policies.py' + } + } + } + } + } + stage('Finalize Cleanup') { + steps { + sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + deleteDir() + } + } + } + post { + always { + deleteDir() + } + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """, + subject: msg, + to: "${contact2}" + } + } + } +} diff --git a/jenkins/clouds/azure/daily/policies/run_policies.py b/jenkins/clouds/azure/daily/policies/run_policies.py new file mode 100644 index 00000000..5a8d7abe --- /dev/null +++ b/jenkins/clouds/azure/daily/policies/run_policies.py @@ -0,0 +1,105 @@ +import os +from ast import literal_eval + +account_name = os.environ['account_name'] +AZURE_CLIENT_SECRET = os.environ['client_secret'] +AZURE_TENANT_ID = os.environ['tenant_id'] +AZURE_ACCOUNT_ID = os.environ['account_id'] +AZURE_CLIENT_ID = os.environ['client_id'] +AZURE_SUBSCRIPTION_ID = os.environ['subscription_id'] +days_to_delete_resource = os.environ.get('days_to_delete_resource', 7) +LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] +LOGS = os.environ.get('LOGS', 'logs') +ES_HOST = os.environ['ES_HOST'] +ES_PORT = os.environ['ES_PORT'] +CLOUD_GOVERNANCE = "quay.io/ebattat/cloud-governance:latest" + + +def get_policies(file_type: str = '.py', exclude_policies: list = None): + """ + This method return a list of policies name without extension, that can filter by type + @return: list of custodian policies name + """ + exclude_policies = [] if not exclude_policies else exclude_policies + custodian_policies = [] + root_folder = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))) + policies_path = os.path.join(root_folder, 'cloud_governance', 'policy', 'azure') + for (_, _, filenames) in os.walk(policies_path): + for filename in filenames: + if not filename.startswith('__') and filename.endswith(file_type): + if filename.split('.')[0] not in exclude_policies: + if not file_type: + custodian_policies.append(os.path.splitext(filename)[0]) + elif file_type and file_type in filename: + custodian_policies.append(os.path.splitext(filename)[0]) + return custodian_policies + + +GLOBAL_COST_POLICIES = ['cost_billing_reports'] +available_policies = get_policies(exclude_policies=GLOBAL_COST_POLICIES) + + +# # available_policies: Run policies in dry_run="yes" mode + + +def run_cmd(cmd: str): + """ + This method runs the shell command + :param cmd: + :type cmd: + :return: + :rtype: + """ + os.system(cmd) + + +def get_container_cmd(env_dict: dict): + env_list = ' '.join(list(map(lambda item: f'-e {item[0]}="{item[1]}"', env_dict.items()))) + container_name = "cloud-governance" + container_run_cmd = f"""podman run --rm --name "{container_name}" --net="host" {env_list} {CLOUD_GOVERNANCE}""" + return container_run_cmd + + +policies_in_action = os.environ.get('POLICIES_IN_ACTION', []) +if isinstance(policies_in_action, str): + policies_in_action = literal_eval(policies_in_action) +policies_not_action = list(set(available_policies) - set(policies_in_action)) + + +container_env_dict = { + "AZURE_CLIENT_SECRET": AZURE_CLIENT_SECRET, + "AZURE_TENANT_ID": AZURE_TENANT_ID, + "AZURE_ACCOUNT_ID": AZURE_ACCOUNT_ID, + "AZURE_CLIENT_ID": AZURE_CLIENT_ID, + "AZURE_SUBSCRIPTION_ID": AZURE_SUBSCRIPTION_ID, + "account": account_name, + "PUBLIC_CLOUD_NAME": "Azure", + "dry_run": "yes", + "LDAP_HOST_NAME": LDAP_HOST_NAME, + "DAYS_TO_DELETE_RESOURCE": days_to_delete_resource, + "es_host": ES_HOST, "es_port": ES_PORT, + "MANAGER_EMAIL_ALERT": "False", "EMAIL_ALERT": "False", "log_level": "INFO", + 'DAYS_TO_TAKE_ACTION': days_to_delete_resource, +} + + +def run_policies(policies: list, dry_run: str = 'yes'): + container_env_dict.update({}) + for policy in policies: + container_env_dict.update({'dry_run': dry_run, 'policy': policy}) + container_cmd = get_container_cmd(container_env_dict) + run_cmd(container_cmd) + + +# Running the polices in dry_run=yes + +run_cmd(f"echo Running the cloud_governance policies with dry_run=yes") +run_cmd(f"echo Polices list: {policies_not_action}") +run_policies(policies=policies_not_action) + +# Running the polices in dry_run=no + +run_cmd('echo "Running the CloudGovernance policies with dry_run=no" ') +run_cmd(f"echo Polices list: {policies_in_action}") +run_policies(policies=policies_in_action, dry_run='no')