diff --git a/README.md b/README.md index 9e0cc65c..7aac14e8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![PyPI Latest Release](https://img.shields.io/pypi/v/cloud-governance.svg)](https://pypi.org/project/cloud-governance/) -[![Container Repository on Quay](https://quay.io/repository/projectquay/quay/status "Container Repository on Quay")](https://quay.io/repository/ebattat/cloud-governance?tab=tags) +[![Container Repository on Quay](https://quay.io/repository/projectquay/quay/status "Container Repository on Quay")](https://quay.io/repository/cloud-governance/cloud-governance?tab=tags) [![Actions Status](https://github.com/redhat-performance/cloud-governance/actions/workflows/Build.yml/badge.svg)](https://github.com/redhat-performance/cloud-governance/actions)[![Coverage Status](https://coveralls.io/repos/github/redhat-performance/cloud-governance/badge.svg?branch=main)](https://coveralls.io/github/redhat-performance/cloud-governance?branch=main) [![Documentation Status](https://readthedocs.org/projects/cloud-governance/badge/?version=latest)](https://cloud-governance.readthedocs.io/en/latest/?badge=latest) [![python](https://img.shields.io/pypi/pyversions/cloud-governance.svg?color=%2334D058)](https://pypi.org/project/cloud-governance) @@ -111,7 +111,7 @@ _**Table of Contents**_ ```sh # Need to run it with root privileges -sudo podman pull quay.io/ebattat/cloud-governance +sudo podman pull quay.io/cloud-governance/cloud-governance ``` #### Environment variables description: @@ -180,44 +180,44 @@ GOOGLE_APPLICATION_CREDENTIALS=$pwd/service_account.json ```sh # policy=instance_idle -sudo podman run --rm --name cloud-governance -e policy="instance_idle" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="instance_idle" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=instance_run -sudo podman run --rm --name cloud-governance -e policy="instance_run" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="instance_run" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # select policy ['ec2_stop', 's3_inactive', 'empty_roles', 'ip_unattached', 'unused_nat_gateway', 'zombie_snapshots'] -sudo podman run --rm --name cloud-governance -e policy="policy" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="policy" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=ebs_unattached -sudo podman run --rm --name cloud-governance -e policy="ebs_unattached" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="ebs_unattached" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=ebs_in_use -sudo podman run --rm --name cloud-governance -e policy="ebs_in_use" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="ebs_in_use" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=zombie_cluster_resource -sudo podman run --rm --name cloud-governance -e policy="zombie_cluster_resource" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e resource="zombie_cluster_elastic_ip" -e cluster_tag="kubernetes.io/cluster/test-pd9qq" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="zombie_cluster_resource" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e resource="zombie_cluster_elastic_ip" -e cluster_tag="kubernetes.io/cluster/test-pd9qq" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=tag_resources -sudo podman run --rm --name cloud-governance -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e tag_operation="read/update/delete" -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e tag_operation="read/update/delete" -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/cloud-governance/cloud-governance" # policy=tag_non_cluster -sudo podman run --rm --name cloud-governance -e policy="tag_non_cluster" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e tag_operation="read/update/delete" -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="tag_non_cluster" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e tag_operation="read/update/delete" -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/cloud-governance/cloud-governance" # policy=tag_iam_user -sudo podman run --rm --name cloud-governance -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e user_tag_operation="read/update/delete" -e remove_tags="['Environment', 'Test']" -e username="test_username" -e file_name="tag_user.csv" -e log_level="INFO" -v "/home/user/tag_user.csv":"/tmp/tag_user.csv" --privileged "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e user_tag_operation="read/update/delete" -e remove_tags="['Environment', 'Test']" -e username="test_username" -e file_name="tag_user.csv" -e log_level="INFO" -v "/home/user/tag_user.csv":"/tmp/tag_user.csv" --privileged "quay.io/cloud-governance/cloud-governance" # policy=cost_explorer -sudo podman run --rm --name cloud-governance -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e es_host="$elasticsearch_host" -e es_port="$elasticsearch_port" -e es_index="$elasticsearch_index" -e cost_metric=UnblendedCost -e start_date="$start_date" -e end_date="$end_date" -e granularity="DAILY" -e cost_explorer_tags="['User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email']" -e log_level="INFO" "quay.io/ebattat/cloud-governance:latest" -sudo podman run --rm --name cloud-governance -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e es_index="elasticsearch_index" -e cost_metric="UnblendedCost" -e start_date="$start_date" -e end_date="$end_date" -e granularity="DAILY" -e cost_explorer_tags="['User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email']" -e file_name="cost_explorer.txt" -v "/home/cost_explorer.txt":"/tmp/cost_explorer.txt" -e log_level="INFO" "quay.io/ebattat/cloud-governance:latest" +sudo podman run --rm --name cloud-governance -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e es_host="$elasticsearch_host" -e es_port="$elasticsearch_port" -e es_index="$elasticsearch_index" -e cost_metric=UnblendedCost -e start_date="$start_date" -e end_date="$end_date" -e granularity="DAILY" -e cost_explorer_tags="['User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email']" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance:latest" +sudo podman run --rm --name cloud-governance -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e es_index="elasticsearch_index" -e cost_metric="UnblendedCost" -e start_date="$start_date" -e end_date="$end_date" -e granularity="DAILY" -e cost_explorer_tags="['User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email']" -e file_name="cost_explorer.txt" -v "/home/cost_explorer.txt":"/tmp/cost_explorer.txt" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance:latest" # policy=validate_iam_user_tags -sudo podman run --rm --name cloud-governance -e policy="validate_iam_user_tags" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e validate_type="spaces/tags" -e user_tags="['Budget', 'User', 'Owner', 'Manager', 'Environment', 'Project']" -e log_level="INFO" "quay.io/ebattat/cloud-governance:latest" +sudo podman run --rm --name cloud-governance -e policy="validate_iam_user_tags" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e validate_type="spaces/tags" -e user_tags="['Budget', 'User', 'Owner', 'Manager', 'Environment', 'Project']" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance:latest" # policy=gitleaks -sudo podman run --rm --name cloud-governance -e policy="gitleaks" -e git_access_token="$git_access_token" -e git_repo="https://github.com/redhat-performance/cloud-governance" -e several_repos="no" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="gitleaks" -e git_access_token="$git_access_token" -e git_repo="https://github.com/redhat-performance/cloud-governance" -e several_repos="no" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # custom cloud custodian policy (path for custom policy: -v /home/user/custodian_policy:/custodian_policy) -sudo podman run --rm --name cloud-governance -e policy="/custodian_policy/policy.yml" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" -v "/home/user/custodian_policy":"/custodian_policy" --privileged "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="/custodian_policy/policy.yml" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" -v "/home/user/custodian_policy":"/custodian_policy" --privileged "quay.io/cloud-governance/cloud-governance" ``` @@ -225,10 +225,10 @@ sudo podman run --rm --name cloud-governance -e policy="/custodian_policy/policy ```sh # policy=tag_baremetal -podman run --rm --name cloud-governance -e policy="tag_baremetal" -e account="$account" -e IBM_API_USERNAME="$IBM_API_USERNAME" -e IBM_API_KEY="$IBM_API_KEY" -e SPREADSHEET_ID="$SPREADSHEET_ID" -e GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" -v $GOOGLE_APPLICATION_CREDENTIALS:$GOOGLE_APPLICATION_CREDENTIALS -e LDAP_USER_HOST="$LDAP_USER_HOST" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/ebattat/cloud-governance:latest" +podman run --rm --name cloud-governance -e policy="tag_baremetal" -e account="$account" -e IBM_API_USERNAME="$IBM_API_USERNAME" -e IBM_API_KEY="$IBM_API_KEY" -e SPREADSHEET_ID="$SPREADSHEET_ID" -e GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" -v $GOOGLE_APPLICATION_CREDENTIALS:$GOOGLE_APPLICATION_CREDENTIALS -e LDAP_USER_HOST="$LDAP_USER_HOST" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/cloud-governance/cloud-governance:latest" # tag=tab_vm -podman run --rm --name cloud-governance -e policy="tag_vm" -e account="$account" -e IBM_API_USERNAME="$IBM_API_USERNAME" -e IBM_API_KEY="$IBM_API_KEY" -e SPREADSHEET_ID="$SPREADSHEET_ID" -e GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" -v $GOOGLE_APPLICATION_CREDENTIALS:$GOOGLE_APPLICATION_CREDENTIALS -e LDAP_USER_HOST="$LDAP_USER_HOST" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/ebattat/cloud-governance:latest" +podman run --rm --name cloud-governance -e policy="tag_vm" -e account="$account" -e IBM_API_USERNAME="$IBM_API_USERNAME" -e IBM_API_KEY="$IBM_API_KEY" -e SPREADSHEET_ID="$SPREADSHEET_ID" -e GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" -v $GOOGLE_APPLICATION_CREDENTIALS:$GOOGLE_APPLICATION_CREDENTIALS -e LDAP_USER_HOST="$LDAP_USER_HOST" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/cloud-governance/cloud-governance:latest" ``` @@ -268,5 +268,5 @@ rm -rf *governance* #### Delete cloud-governance image ```sh -sudo podman rmi quay.io/ebattat/cloud-governance +sudo podman rmi quay.io/cloud-governance/cloud-governance ``` diff --git a/cloud_governance/cloud_resource_orchestration/README.md b/cloud_governance/cloud_resource_orchestration/README.md index 8e58c36b..9fb398a8 100644 --- a/cloud_governance/cloud_resource_orchestration/README.md +++ b/cloud_governance/cloud_resource_orchestration/README.md @@ -2,7 +2,8 @@ This is the process to control costs on public clouds. \ This process requires the data how many days a project will run and estimated_cost. \ -Details are collected from the front end page @[https://cloud-governance.rdu2.scalelab.redhat.com/](https://cloud-governance.rdu2.scalelab.redhat.com/) +Details are collected from the front end +page @[https://cloud-governance.rdu2.scalelab.redhat.com/](https://cloud-governance.rdu2.scalelab.redhat.com/) After filling the form, mail sent to manager for approval after approved your request. Tag your instances with TicketId: #ticket_number. \ Then cloud_governance will start **cloud_resource_orchestration** and monitor your instances. @@ -10,5 +11,5 @@ Then cloud_governance will start **cloud_resource_orchestration** and monitor yo To start **cloud_resource_orchestration** CI run the below podman command ```commandline -podman run --net="host" --rm --name cloud_resource_orchestration -e AWS_DEFAULT_REGION="ap-south-1" -e CLOUD_RESOURCE_ORCHESTRATION="True" -e account="$account" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e PUBLIC_CLOUD_NAME="$PUBLIC_CLOUD_NAME" -e es_host="$ES_HOST" -e es_port="$ES_PORT" -e CRO_ES_INDEX="$CRO_ES_INDEX" -e log_level="INFO" -e LDAP_HOST_NAME="$LDAP_HOST_NAME" -e JIRA_QUEUE="$JIRA_QUEUE" -e JIRA_TOKEN="$JIRA_TOKEN" -e JIRA_USERNAME="$JIRA_USERNAME" -e JIRA_URL="$JIRA_URL" -e CRO_COST_OVER_USAGE="$CRO_COST_OVER_USAGE" -e CRO_PORTAL="$CRO_PORTAL" -e CRO_DEFAULT_ADMINS="$CRO_DEFAULT_ADMINS" -e CRO_REPLACED_USERNAMES="$CRO_REPLACED_USERNAMES" -e CRO_DURATION_DAYS="30" quay.io/ebattat/cloud-governance:latest +podman run --net="host" --rm --name cloud_resource_orchestration -e AWS_DEFAULT_REGION="ap-south-1" -e CLOUD_RESOURCE_ORCHESTRATION="True" -e account="$account" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e PUBLIC_CLOUD_NAME="$PUBLIC_CLOUD_NAME" -e es_host="$ES_HOST" -e es_port="$ES_PORT" -e CRO_ES_INDEX="$CRO_ES_INDEX" -e log_level="INFO" -e LDAP_HOST_NAME="$LDAP_HOST_NAME" -e JIRA_QUEUE="$JIRA_QUEUE" -e JIRA_TOKEN="$JIRA_TOKEN" -e JIRA_USERNAME="$JIRA_USERNAME" -e JIRA_URL="$JIRA_URL" -e CRO_COST_OVER_USAGE="$CRO_COST_OVER_USAGE" -e CRO_PORTAL="$CRO_PORTAL" -e CRO_DEFAULT_ADMINS="$CRO_DEFAULT_ADMINS" -e CRO_REPLACED_USERNAMES="$CRO_REPLACED_USERNAMES" -e CRO_DURATION_DAYS="30" quay.io/cloud-governance/cloud-governance:latest ``` diff --git a/docs/source/installation.md b/docs/source/installation.md index 22cd83bb..0cdd76be 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -1,9 +1,10 @@ # Installation #### Download cloud-governance image from quay.io + ```sh # Need to run it with root privileges -sudo podman pull quay.io/ebattat/cloud-governance +sudo podman pull quay.io/cloud-governance/cloud-governance ``` #### Environment variables description: @@ -13,33 +14,44 @@ sudo podman pull quay.io/ebattat/cloud-governance (mandatory)AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY ##### Policy name: -(mandatory)policy=instance_idle / ec2_run / ebs_unattached / ebs_in_use / tag_cluster_resource / zombie_cluster_resource / tag_ec2_resource + +(mandatory)policy=instance_idle / ec2_run / ebs_unattached / ebs_in_use / tag_cluster_resource / +zombie_cluster_resource / tag_ec2_resource ##### Policy logs output + (mandatory)policy_output=s3://redhat-cloud-governance/logs ##### Cluster or instance name: + (mandatory policy:tag_cluster_resource)resource_name=ocs-test ##### Cluster or instance tags: + (mandatory policy:tag_cluster_resource)mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" ##### gitleaks + (mandatory policy: gitleaks)git_access_token=$git_access_token (mandatory policy: gitleaks)git_repo=https://github.com/redhat-performance/cloud-governance (optional policy: gitleaks)several_repos=yes/no (default = no) ##### Choose a specific region or all for all the regions, default : us-east-2 + (optional)AWS_DEFAULT_REGION=us-east-2/all (default = us-east-2) ##### Choose dry run or not, default yes + (optional)dry_run=yes/no (default = yes) ##### Choose log level, default INFO + (optional)log_level=INFO (default = INFO) #### LDAP hostname to fetch mail records + LDAP_HOST_NAME=ldap.example.com #### Enable Google Drive API in console and create Service account + GOOGLE_APPLICATION_CREDENTIALS=$pwd/service_account.json diff --git a/docs/source/podman.md b/docs/source/podman.md index 242c4727..5d99b4b8 100644 --- a/docs/source/podman.md +++ b/docs/source/podman.md @@ -1,44 +1,45 @@ ## Run AWS Policy Using Podman + ```sh # policy=instance_idle -sudo podman run --rm --name cloud-governance -e policy="instance_idle" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="instance_idle" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=ec2_run -sudo podman run --rm --name cloud-governance -e policy="ec2_run" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="ec2_run" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # select policy ['ec2_stop', 's3_inactive', 'empty_roles', 'ip_unattached', 'nat_gateway_unused', 'zombie_snapshots'] -sudo podman run --rm --name cloud-governance -e policy="policy" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="policy" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=ebs_unattached -sudo podman run --rm --name cloud-governance -e policy="ebs_unattached" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="ebs_unattached" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=ebs_in_use -sudo podman run --rm --name cloud-governance -e policy="ebs_in_use" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="ebs_in_use" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=zombie_cluster_resource -sudo podman run --rm --name cloud-governance -e policy="zombie_cluster_resource" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e resource="zombie_cluster_elastic_ip" -e cluster_tag="kubernetes.io/cluster/test-pd9qq" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="zombie_cluster_resource" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e resource="zombie_cluster_elastic_ip" -e cluster_tag="kubernetes.io/cluster/test-pd9qq" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # policy=tag_resources -sudo podman run --rm --name cloud-governance -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e tag_operation="read/update/delete" -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e tag_operation="read/update/delete" -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/cloud-governance/cloud-governance" # policy=tag_non_cluster -sudo podman run --rm --name cloud-governance -e policy="tag_non_cluster" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e tag_operation="read/update/delete" -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="tag_non_cluster" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e tag_operation="read/update/delete" -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/cloud-governance/cloud-governance" # policy=tag_iam_user -sudo podman run --rm --name cloud-governance -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e user_tag_operation="read/update/delete" -e remove_tags="['Environment', 'Test']" -e username="test_username" -e file_name="tag_user.csv" -e log_level="INFO" -v "/home/user/tag_user.csv":"/tmp/tag_user.csv" --privileged "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e user_tag_operation="read/update/delete" -e remove_tags="['Environment', 'Test']" -e username="test_username" -e file_name="tag_user.csv" -e log_level="INFO" -v "/home/user/tag_user.csv":"/tmp/tag_user.csv" --privileged "quay.io/cloud-governance/cloud-governance" # policy=cost_explorer -sudo podman run --rm --name cloud-governance -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e es_host="$elasticsearch_host" -e es_port="$elasticsearch_port" -e es_index="$elasticsearch_index" -e cost_metric=UnblendedCost -e start_date="$start_date" -e end_date="$end_date" -e granularity="DAILY" -e cost_explorer_tags="['User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email']" -e log_level="INFO" "quay.io/ebattat/cloud-governance:latest" -sudo podman run --rm --name cloud-governance -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e es_index="elasticsearch_index" -e cost_metric="UnblendedCost" -e start_date="$start_date" -e end_date="$end_date" -e granularity="DAILY" -e cost_explorer_tags="['User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email']" -e file_name="cost_explorer.txt" -v "/home/cost_explorer.txt":"/tmp/cost_explorer.txt" -e log_level="INFO" "quay.io/ebattat/cloud-governance:latest" +sudo podman run --rm --name cloud-governance -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e es_host="$elasticsearch_host" -e es_port="$elasticsearch_port" -e es_index="$elasticsearch_index" -e cost_metric=UnblendedCost -e start_date="$start_date" -e end_date="$end_date" -e granularity="DAILY" -e cost_explorer_tags="['User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email']" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance:latest" +sudo podman run --rm --name cloud-governance -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e es_index="elasticsearch_index" -e cost_metric="UnblendedCost" -e start_date="$start_date" -e end_date="$end_date" -e granularity="DAILY" -e cost_explorer_tags="['User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email']" -e file_name="cost_explorer.txt" -v "/home/cost_explorer.txt":"/tmp/cost_explorer.txt" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance:latest" # policy=validate_iam_user_tags -sudo podman run --rm --name cloud-governance -e policy="validate_iam_user_tags" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e validate_type="spaces/tags" -e user_tags="['Budget', 'User', 'Owner', 'Manager', 'Environment', 'Project']" -e log_level="INFO" "quay.io/ebattat/cloud-governance:latest" +sudo podman run --rm --name cloud-governance -e policy="validate_iam_user_tags" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e validate_type="spaces/tags" -e user_tags="['Budget', 'User', 'Owner', 'Manager', 'Environment', 'Project']" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance:latest" # policy=gitleaks -sudo podman run --rm --name cloud-governance -e policy="gitleaks" -e git_access_token="$git_access_token" -e git_repo="https://github.com/redhat-performance/cloud-governance" -e several_repos="no" -e log_level="INFO" "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="gitleaks" -e git_access_token="$git_access_token" -e git_repo="https://github.com/redhat-performance/cloud-governance" -e several_repos="no" -e log_level="INFO" "quay.io/cloud-governance/cloud-governance" # custom cloud custodian policy (path for custom policy: -v /home/user/custodian_policy:/custodian_policy) -sudo podman run --rm --name cloud-governance -e policy="/custodian_policy/policy.yml" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" -v "/home/user/custodian_policy":"/custodian_policy" --privileged "quay.io/ebattat/cloud-governance" +sudo podman run --rm --name cloud-governance -e policy="/custodian_policy/policy.yml" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_DEFAULT_REGION="us-east-2" -e dry_run="yes" -e policy_output="s3://bucket/logs" -e log_level="INFO" -v "/home/user/custodian_policy":"/custodian_policy" --privileged "quay.io/cloud-governance/cloud-governance" ``` @@ -46,9 +47,9 @@ sudo podman run --rm --name cloud-governance -e policy="/custodian_policy/policy ```sh # policy=tag_baremetal -podman run --rm --name cloud-governance -e policy="tag_baremetal" -e account="$account" -e IBM_API_USERNAME="$IBM_API_USERNAME" -e IBM_API_KEY="$IBM_API_KEY" -e SPREADSHEET_ID="$SPREADSHEET_ID" -e GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" -v $GOOGLE_APPLICATION_CREDENTIALS:$GOOGLE_APPLICATION_CREDENTIALS -e LDAP_USER_HOST="$LDAP_USER_HOST" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/ebattat/cloud-governance:latest" +podman run --rm --name cloud-governance -e policy="tag_baremetal" -e account="$account" -e IBM_API_USERNAME="$IBM_API_USERNAME" -e IBM_API_KEY="$IBM_API_KEY" -e SPREADSHEET_ID="$SPREADSHEET_ID" -e GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" -v $GOOGLE_APPLICATION_CREDENTIALS:$GOOGLE_APPLICATION_CREDENTIALS -e LDAP_USER_HOST="$LDAP_USER_HOST" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/cloud-governance/cloud-governance:latest" # tag=tab_vm -podman run --rm --name cloud-governance -e policy="tag_vm" -e account="$account" -e IBM_API_USERNAME="$IBM_API_USERNAME" -e IBM_API_KEY="$IBM_API_KEY" -e SPREADSHEET_ID="$SPREADSHEET_ID" -e GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" -v $GOOGLE_APPLICATION_CREDENTIALS:$GOOGLE_APPLICATION_CREDENTIALS -e LDAP_USER_HOST="$LDAP_USER_HOST" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/ebattat/cloud-governance:latest" +podman run --rm --name cloud-governance -e policy="tag_vm" -e account="$account" -e IBM_API_USERNAME="$IBM_API_USERNAME" -e IBM_API_KEY="$IBM_API_KEY" -e SPREADSHEET_ID="$SPREADSHEET_ID" -e GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" -v $GOOGLE_APPLICATION_CREDENTIALS:$GOOGLE_APPLICATION_CREDENTIALS -e LDAP_USER_HOST="$LDAP_USER_HOST" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" "quay.io/cloud-governance/cloud-governance:latest" ``` diff --git a/docs/source/postinstallation.md b/docs/source/postinstallation.md index 6a92ba46..58c8a698 100644 --- a/docs/source/postinstallation.md +++ b/docs/source/postinstallation.md @@ -1,6 +1,7 @@ # Post Installation #### Delete cloud-governance image + ```sh -sudo podman rmi quay.io/ebattat/cloud-governance +sudo podman rmi quay.io/cloud-governance/cloud-governance ``` diff --git a/docs/source/tagging.md b/docs/source/tagging.md index d6658f77..b770b5c2 100644 --- a/docs/source/tagging.md +++ b/docs/source/tagging.md @@ -2,33 +2,34 @@ This feature help you tagging your account for cluster, non cluster resource and IAM user. - 1. Update all the users in account with permanent tags: [user_data_csv=read/write]
- we create a simple way to read all user into csv, update tags in the csv by columns,delete the tags by specific keys and run podman to update it in IAM. - 1. read - generates csv file with all the users and tags of users. [ manually update the list ] - 2. update - Updates the tags of users in IAM - 3. delete - Delete the specific tags in IAM user + we create a simple way to read all user into csv, update tags in the csv by columns,delete the tags by specific keys + and run podman to update it in IAM. + 1. read - generates csv file with all the users and tags of users. [ manually update the list ] + 2. update - Updates the tags of users in IAM + 3. delete - Delete the specific tags in IAM user ```sh # policy=tag_iam_user -sudo podman run --rm --name cloud-governance -e policy=tag_iam_user -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e user_tag_operation=read/update/delete -e remove_tags="['Environment', 'Test']" -e username=test_username -e file_name=tag_user.csv -e log_level=INFO -v /home/user/tag_user.csv:/tmp/tag_user.csv --privileged quay.io/ebattat/cloud-governance +sudo podman run --rm --name cloud-governance -e policy=tag_iam_user -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e user_tag_operation=read/update/delete -e remove_tags="['Environment', 'Test']" -e username=test_username -e file_name=tag_user.csv -e log_level=INFO -v /home/user/tag_user.csv:/tmp/tag_user.csv --privileged quay.io/cloud-governance/cloud-governance ``` 2. Update all the resources Cluster/Non-Cluster by create user tags.
It will update all the non tags resource per region [Limit: only if the resource was created in the last 90 days]. - 1. mandatory_tags: it adds the tags to the resource. - 2. tag_operation: read/update/delete to perform the operation + 1. mandatory_tags: it adds the tags to the resource. + 2. tag_operation: read/update/delete to perform the operation ```sh # policy=tag_resources -sudo podman run --rm --name cloud-governance -e policy=tag_resources -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=us-east-2 -e tag_operation=read/update/delete -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level=INFO -v /etc/localtime:/etc/localtime quay.io/ebattat/cloud-governance +sudo podman run --rm --name cloud-governance -e policy=tag_resources -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=us-east-2 -e tag_operation=read/update/delete -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level=INFO -v /etc/localtime:/etc/localtime quay.io/cloud-governance/cloud-governance ``` 3. Update all the non-cluster resources by create user tags and mandatory tags - It will update all the non tags of non-cluster resource per region [Limit: only if the resource was created in the last 90 days]. - 1. mandatory_tags: it adds the tags of the resource if it doesn't have any data on user. + It will update all the non tags of non-cluster resource per + region [Limit: only if the resource was created in the last 90 days]. + 1. mandatory_tags: it adds the tags of the resource if it doesn't have any data on user. ```sh # policy=tag_non_cluster -sudo podman run --rm --name cloud-governance -e policy=tag_non_cluster -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=us-east-2 -e dtag_operation=read/delete/update -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level=INFO -v /etc/localtime:/etc/localtime quay.io/ebattat/cloud-governance +sudo podman run --rm --name cloud-governance -e policy=tag_non_cluster -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=us-east-2 -e dtag_operation=read/delete/update -e mandatory_tags="{'Owner': 'Name','Email': 'name@redhat.com','Purpose': 'test'}" -e log_level=INFO -v /etc/localtime:/etc/localtime quay.io/cloud-governance/cloud-governance ``` diff --git a/jenkins/cloud_resource_orchestration/Jenkinsfile b/jenkins/cloud_resource_orchestration/Jenkinsfile index f5e92ef2..d934b041 100644 --- a/jenkins/cloud_resource_orchestration/Jenkinsfile +++ b/jenkins/cloud_resource_orchestration/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_ACCESS_KEY_ID_DELETE_PERF = credentials('cloud-governance-aws-access-key-id-delete-perf') AWS_SECRET_ACCESS_KEY_DELETE_PERF = credentials('cloud-governance-aws-secret-access-key-delete-perf') AWS_ACCESS_KEY_ID_DELETE_PSAP = credentials('cloud-governance-aws-access-key-id-delete-psap') @@ -47,7 +48,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/athiru/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run the CloudResourceOrchestration') { @@ -57,7 +58,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/athiru/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/athiru/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/cloud_resource_orchestration/run_cloud_resource_orchestration.py b/jenkins/cloud_resource_orchestration/run_cloud_resource_orchestration.py index 5feefabc..1f512b91 100644 --- a/jenkins/cloud_resource_orchestration/run_cloud_resource_orchestration.py +++ b/jenkins/cloud_resource_orchestration/run_cloud_resource_orchestration.py @@ -25,6 +25,8 @@ S3_RESULTS_PATH = os.environ['S3_RESULTS_PATH'] ATHENA_DATABASE_NAME = os.environ['ATHENA_DATABASE_NAME'] ATHENA_TABLE_NAME = os.environ['ATHENA_TABLE_NAME'] +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') es_index = CLOUD_RESOURCE_ORCHESTRATION_INDEX @@ -40,7 +42,8 @@ input_vars_to_container = [{'account': 'perf-dept', 'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID_DELETE_PERF, 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY_DELETE_PERF, 'PUBLIC_CLOUD_NAME': 'AWS'}, {'account': 'perf-scale', 'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE, - 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE, 'PUBLIC_CLOUD_NAME': 'AWS'}, + 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE, + 'PUBLIC_CLOUD_NAME': 'AWS'}, {'account': 'psap', 'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID_DELETE_PSAP, 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY_DELETE_PSAP, 'PUBLIC_CLOUD_NAME': 'AWS'}] @@ -68,8 +71,8 @@ for input_vars in input_vars_to_container: os.system(f"""echo Running on Account {input_vars.get("account").upper()}""") envs = list(map(combine_vars, input_vars.items())) - os.system(f"""podman run --net="host" --rm --name cloud_resource_orchestration -e CLOUD_RESOURCE_ORCHESTRATION="True" -e EMAIL_ALERT="True" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} quay.io/ebattat/cloud-governance:latest""") - + os.system( + f"""podman run --net="host" --rm --name cloud_resource_orchestration -e CLOUD_RESOURCE_ORCHESTRATION="True" -e EMAIL_ALERT="True" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") AZURE_ACCOUNT_ID = os.environ['AZURE_ACCOUNT_ID'] AZURE_CLIENT_SECRET = os.environ['AZURE_CLIENT_SECRET'] @@ -88,5 +91,5 @@ azure_cro_env.update(common_env_vars) envs = list(map(combine_vars, azure_cro_env.items())) azure_cro = """ podman run --net="host" --rm --name cloud_resource_orchestration """ -azure_cro += f" -e {' -e '.join(envs)} quay.io/ebattat/cloud-governance:latest" +azure_cro += f" -e {' -e '.join(envs)} {QUAY_CLOUD_GOVERNANCE_REPOSITORY}" os.system(azure_cro) diff --git a/jenkins/clouds/aws/daily/cost_explorer/Jenkinsfile b/jenkins/clouds/aws/daily/cost_explorer/Jenkinsfile index 323498bf..75c70811 100644 --- a/jenkins/clouds/aws/daily/cost_explorer/Jenkinsfile +++ b/jenkins/clouds/aws/daily/cost_explorer/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_ACCESS_KEY_ID_PERF = credentials('cloud-governance-aws-access-key-id-perf') AWS_SECRET_ACCESS_KEY_PERF = credentials('cloud-governance-aws-secret-access-key-perf') AWS_ACCESS_KEY_ID_DELETE_PERF = credentials('cloud-governance-aws-access-key-id-delete-perf') @@ -42,7 +43,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Upload ElasticSearch') { @@ -52,7 +53,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/aws/daily/cost_explorer/run_upload_es.py b/jenkins/clouds/aws/daily/cost_explorer/run_upload_es.py index 7389cfdc..097973bd 100644 --- a/jenkins/clouds/aws/daily/cost_explorer/run_upload_es.py +++ b/jenkins/clouds/aws/daily/cost_explorer/run_upload_es.py @@ -22,6 +22,8 @@ special_user_mails = os.environ['CLOUD_GOVERNANCE_SPECIAL_USER_MAILS'] COST_SPREADSHEET_ID = os.environ['COST_SPREADSHEET_ID'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') es_index_perf = 'cloud-governance-cost-explorer-perf' es_index_psap = 'cloud-governance-cost-explorer-psap' @@ -34,25 +36,19 @@ cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost granularity = 'DAILY' # DAILY/MONTHLY/HOURLY os.system( - f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_psap}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_psap}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index={es_index_perf_scale} -e cost_explorer_tags="{cost_tags}" -e granularity={granularity} -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") - -# @Todo, remove if cost explorer working properly - -# os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") -# os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") -# os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index={es_index_perf_scale} -e cost_explorer_tags="{cost_tags}" -e granularity={granularity} -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") es_index_global = 'cloud-governance-cost-explorer-perf-global-cost' os.system( - f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") input_vars_to_container = [{'account': 'perf-dept', 'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID_DELETE_PERF, 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY_DELETE_PERF}, @@ -68,4 +64,4 @@ for input_vars in input_vars_to_container: envs = list(map(combine_vars, input_vars.items())) os.system( - f"""podman run --rm --name cloud-governance -e policy="cost_billing_reports" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" quay.io/ebattat/cloud-governance:latest""") + f"""podman run --rm --name cloud-governance -e policy="cost_billing_reports" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile b/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile index bf52b725..b67f9c56 100644 --- a/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile +++ b/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_ACCESS_KEY_ID_DELETE_PERF = credentials('cloud-governance-aws-access-key-id-delete-perf') AWS_SECRET_ACCESS_KEY_DELETE_PERF = credentials('cloud-governance-aws-secret-access-key-delete-perf') ES_HOST = credentials('cloud-governance-es-host') @@ -42,7 +43,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run the AWS Cost Reports') { @@ -52,7 +53,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py index b8a78e1e..48705140 100644 --- a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py +++ b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py @@ -15,6 +15,8 @@ S3_RESULTS_PATH = os.environ['S3_RESULTS_PATH'] ATHENA_DATABASE_NAME = os.environ['ATHENA_DATABASE_NAME'] ATHENA_TABLE_NAME = os.environ['ATHENA_TABLE_NAME'] +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') # Cloudability env variables @@ -26,7 +28,6 @@ APPITO_KEY_SECRET = os.environ['APPITO_KEY_SECRET'] APPITO_ENVID = os.environ['APPITO_ENVID'] - os.system('echo "Updating the Org level cost billing reports"') # Cost Explorer upload to ElasticSearch @@ -42,7 +43,7 @@ common_input_vars['es_index'] = 'cloud-governance-clouds-billing-reports' common_envs = list(map(combine_vars, common_input_vars.items())) os.system( - f"""podman run --rm --name cloud-governance -e policy="cost_explorer_payer_billings" -e AWS_ACCOUNT_ROLE="{AWS_ACCOUNT_ROLE}" -e account="PERF-DEPT" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" quay.io/ebattat/cloud-governance:latest""") + f"""podman run --rm --name cloud-governance -e policy="cost_explorer_payer_billings" -e AWS_ACCOUNT_ROLE="{AWS_ACCOUNT_ROLE}" -e account="PERF-DEPT" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system('echo "Run the Spot Analysis report over the account using AWS Athena"') os.system(f"""podman run --rm --name cloud-governance -e policy="spot_savings_analysis" -e account="pnt-payer" \ @@ -53,9 +54,8 @@ -e S3_RESULTS_PATH="{S3_RESULTS_PATH}" \ -e ATHENA_DATABASE_NAME="{ATHENA_DATABASE_NAME}" \ -e ATHENA_TABLE_NAME="{ATHENA_TABLE_NAME}" \ -quay.io/ebattat/cloud-governance:latest""") +{QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") -CLOUD_GOVERNANCE_IMAGE = "quay.io/ebattat/cloud-governance:latest" CONTAINER_NAME = "cloud-governance" COST_ES_INDEX = "cloud-governance-clouds-billing-reports" CLOUDABILITY_POLICY = 'cloudability_cost_reports' @@ -86,7 +86,7 @@ def generate_shell_cmd(policy: str, env_variables: dict, mounted_volumes: str = """ inject_container_envs = ' '.join(list(map(lambda item: f'-e {item[0]}="{item[1]}"', env_variables.items()))) return (f'podman run --rm --name {CONTAINER_NAME} -e policy="{policy}" {inject_container_envs} {mounted_volumes} ' - f'{CLOUD_GOVERNANCE_IMAGE}') + f'{QUAY_CLOUD_GOVERNANCE_REPOSITORY}') common_env_vars = { diff --git a/jenkins/clouds/aws/daily/policies/Jenkinsfile b/jenkins/clouds/aws/daily/policies/Jenkinsfile index 2c2cbbfe..fac20b69 100644 --- a/jenkins/clouds/aws/daily/policies/Jenkinsfile +++ b/jenkins/clouds/aws/daily/policies/Jenkinsfile @@ -11,6 +11,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') POLICIES_IN_ACTION = '["instance_idle", "ec2_stop", "unattached_volume", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles", "zombie_cluster_resource"]' AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') @@ -37,7 +38,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Daily Policies') { @@ -58,7 +59,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/aws/daily/policies/run_policies.py b/jenkins/clouds/aws/daily/policies/run_policies.py index 5690f8fe..8c2d7cb6 100644 --- a/jenkins/clouds/aws/daily/policies/run_policies.py +++ b/jenkins/clouds/aws/daily/policies/run_policies.py @@ -18,8 +18,9 @@ GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] GITHUB_TOKEN = os.environ['GITHUB_TOKEN'] -CLOUD_GOVERNANCE_IMAGE = "quay.io/ebattat/cloud-governance:latest" ADMIN_MAIL_LIST = os.environ.get('ADMIN_MAIL_LIST', '') +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') def get_policies(file_type: str = '.py', exclude_policies: list = None): @@ -68,7 +69,7 @@ def get_container_cmd(env_dict: dict): env_list = ' '.join(list(map(lambda item: f'-e {item[0]}="{item[1]}"', env_dict.items()))) container_name = "cloud-governance" container_run_cmd = f""" -podman run --rm --name "{container_name}" --net="host" {env_list} quay.io/ebattat/cloud-governance:latest +podman run --rm --name "{container_name}" --net="host" {env_list} {QUAY_CLOUD_GOVERNANCE_REPOSITORY} """ return container_run_cmd @@ -117,7 +118,6 @@ def run_policies(policies: list, dry_run: str = 'yes'): run_cmd(f"echo Polices list: {policies_not_action}") run_policies(policies=policies_not_action) - # Running the polices in dry_run=no run_cmd('echo "Running the CloudGovernance policies with dry_run=no" ') @@ -127,18 +127,21 @@ def run_policies(policies: list, dry_run: str = 'yes'): # Update AWS IAM User tags from the spreadsheet run_cmd(f"""echo "Running the tag_iam_user" """) -run_cmd(f"""podman run --rm --name cloud-governance --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e account_admin="{account_admin}" -e special_user_mails="{special_user_mails}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +run_cmd( + f"""podman run --rm --name cloud-governance --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e account_admin="{account_admin}" -e special_user_mails="{special_user_mails}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") # Running the trust advisor reports, data dumped into default index - cloud-governance-policy-es-index -run_cmd(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +run_cmd( + f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") # Git-leaks run on GitHub not related to any aws account run_cmd("echo Run Git-leaks") region = 'us-east-1' policy = 'gitleaks' -run_cmd(f"""podman run --rm --name cloud-governance -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e git_access_token="{GITHUB_TOKEN}" -e git_repo="https://github.com/redhat-performance" -e several_repos="yes" -e policy_output="s3://{s3_bucket}/{LOGS}/$region" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") - +run_cmd( + f"""podman run --rm --name cloud-governance -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e git_access_token="{GITHUB_TOKEN}" -e git_repo="https://github.com/redhat-performance" -e several_repos="yes" -e policy_output="s3://{s3_bucket}/{LOGS}/$region" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") -run_cmd(f"""podman run --rm --name cloud-governance --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" {CLOUD_GOVERNANCE_IMAGE}""") +run_cmd( + f"""podman run --rm --name cloud-governance --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/aws/hourly/tagging/Jenkinsfile b/jenkins/clouds/aws/hourly/tagging/Jenkinsfile index 315eff75..9a783468 100644 --- a/jenkins/clouds/aws/hourly/tagging/Jenkinsfile +++ b/jenkins/clouds/aws/hourly/tagging/Jenkinsfile @@ -10,6 +10,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_ACCESS_KEY_ID_DELETE_PERF = credentials('cloud-governance-aws-access-key-id-delete-perf') AWS_SECRET_ACCESS_KEY_DELETE_PERF = credentials('cloud-governance-aws-secret-access-key-delete-perf') AWS_ACCESS_KEY_ID_DELETE_PSAP = credentials('cloud-governance-aws-access-key-id-delete-psap') @@ -27,7 +28,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -37,7 +38,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/aws/hourly/tagging/tagging.py b/jenkins/clouds/aws/hourly/tagging/tagging.py index a34123ac..6431fcfd 100644 --- a/jenkins/clouds/aws/hourly/tagging/tagging.py +++ b/jenkins/clouds/aws/hourly/tagging/tagging.py @@ -1,13 +1,13 @@ - import os - AWS_ACCESS_KEY_ID_DELETE_PERF = os.environ['AWS_ACCESS_KEY_ID_DELETE_PERF'] AWS_SECRET_ACCESS_KEY_DELETE_PERF = os.environ['AWS_SECRET_ACCESS_KEY_DELETE_PERF'] AWS_ACCESS_KEY_ID_DELETE_PSAP = os.environ['AWS_ACCESS_KEY_ID_DELETE_PSAP'] AWS_SECRET_ACCESS_KEY_DELETE_PSAP = os.environ['AWS_SECRET_ACCESS_KEY_DELETE_PSAP'] AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE = os.environ['AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE'] AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE = os.environ['AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE'] +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') LOGS = os.environ.get('LOGS', 'logs') @@ -16,9 +16,14 @@ mandatory_tags_perf_scale = {'Budget': 'PERF-SCALE'} print('Run AWS tagging policy pre active region') -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1'] +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', + 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1', + 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1'] for region in regions: - os.system(f"""podman run --rm --name cloud-governance -e account="perf" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_perf}" -e log_level="INFO" -e "AWS_MAX_ATTEMPTS"="5" -e "AWS_RETRY_MODE"="standard" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") - os.system(f"""podman run --rm --name cloud-governance -e account="psap" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_psap}" -e log_level="INFO" -e "AWS_MAX_ATTEMPTS"="5" -e "AWS_RETRY_MODE"="standard" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") - os.system(f"""podman run --rm --name cloud-governance -e account="perf-scale" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_perf_scale}" -e "AWS_MAX_ATTEMPTS"="5" -e "AWS_RETRY_MODE"="standard" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance -e account="perf" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_perf}" -e log_level="INFO" -e "AWS_MAX_ATTEMPTS"="5" -e "AWS_RETRY_MODE"="standard" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + os.system( + f"""podman run --rm --name cloud-governance -e account="psap" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_psap}" -e log_level="INFO" -e "AWS_MAX_ATTEMPTS"="5" -e "AWS_RETRY_MODE"="standard" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + os.system( + f"""podman run --rm --name cloud-governance -e account="perf-scale" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_perf_scale}" -e "AWS_MAX_ATTEMPTS"="5" -e "AWS_RETRY_MODE"="standard" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/aws/monthly/Jenkinsfile b/jenkins/clouds/aws/monthly/Jenkinsfile index bb08b418..9ee1464d 100644 --- a/jenkins/clouds/aws/monthly/Jenkinsfile +++ b/jenkins/clouds/aws/monthly/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') TO_MAIL = credentials('cloud-governance-to-mail') @@ -23,7 +24,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Monthly Policies') { @@ -33,7 +34,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/aws/monthly/run_monthly_policies.py b/jenkins/clouds/aws/monthly/run_monthly_policies.py index b1ea809f..ca6400f6 100644 --- a/jenkins/clouds/aws/monthly/run_monthly_policies.py +++ b/jenkins/clouds/aws/monthly/run_monthly_policies.py @@ -1,13 +1,15 @@ - import os ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] TO_MAIL = os.environ['TO_MAIL'] CC_MAIL = os.environ['CC_MAIL'] +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') LOGS = os.environ.get('LOGS', 'logs') # RUN AWS Monthly Policies print("Run AWS Monthly Policies") -os.system(f"""podman run --rm --name cloud-governance --net="host" -e policy="monthly_report" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e to_mail="{TO_MAIL}" -e cc_mail="{CC_MAIL}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance --net="host" -e policy="monthly_report" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e to_mail="{TO_MAIL}" -e cc_mail="{CC_MAIL}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/aws/weekly/cost_over_usage/Jenkinsfile b/jenkins/clouds/aws/weekly/cost_over_usage/Jenkinsfile index 70bbbcf9..9ee2cc4b 100644 --- a/jenkins/clouds/aws/weekly/cost_over_usage/Jenkinsfile +++ b/jenkins/clouds/aws/weekly/cost_over_usage/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_ACCESS_KEY_ID_DELETE_PERF = credentials('cloud-governance-aws-access-key-id-delete-perf') AWS_SECRET_ACCESS_KEY_DELETE_PERF = credentials('cloud-governance-aws-secret-access-key-delete-perf') BUCKET_PERF = credentials('cloud-governance-bucket-perf') @@ -34,7 +35,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Upload ElasticSearch') { @@ -44,7 +45,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/aws/weekly/cost_over_usage/run_upload_es.py b/jenkins/clouds/aws/weekly/cost_over_usage/run_upload_es.py index 43132449..d1fe2728 100644 --- a/jenkins/clouds/aws/weekly/cost_over_usage/run_upload_es.py +++ b/jenkins/clouds/aws/weekly/cost_over_usage/run_upload_es.py @@ -1,7 +1,5 @@ - import os - AWS_ACCESS_KEY_ID_DELETE_PERF = os.environ['AWS_ACCESS_KEY_ID_DELETE_PERF'] AWS_SECRET_ACCESS_KEY_DELETE_PERF = os.environ['AWS_SECRET_ACCESS_KEY_DELETE_PERF'] AWS_ACCESS_KEY_ID_DELETE_PSAP = os.environ['AWS_ACCESS_KEY_ID_DELETE_PSAP'] @@ -16,12 +14,16 @@ LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] special_user_mails = os.environ['CLOUD_GOVERNANCE_SPECIAL_USER_MAILS'] IGNORE_MAILS = os.environ['IGNORE_MAILS'] - +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') es_index_perf = 'cloud-governance-cost-explorer-perf' es_index_psap = 'cloud-governance-cost-explorer-psap' es_index_perf_scale = 'cloud-governance-cost-explorer-perf-scale' -os.system(f"""podman run --rm --name cloud-governance --net="host" -e account="perf-dept" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf}-user" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e log_level="INFO" -e policy_output="{BUCKET_PERF}" quay.io/ebattat/cloud-governance:latest""") -os.system(f"""podman run --rm --name cloud-governance --net="host" -e account="psap" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_psap}-user" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e log_level="INFO" -e policy_output="{BUCKET_PSAP}" quay.io/ebattat/cloud-governance:latest""") -os.system(f"""podman run --rm --name cloud-governance --net="host" -e account="perf-scale" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf_scale}-user" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e policy_output="{BUCKET_PERF_SCALE}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance --net="host" -e account="perf-dept" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf}-user" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e log_level="INFO" -e policy_output="{BUCKET_PERF}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") +os.system( + f"""podman run --rm --name cloud-governance --net="host" -e account="psap" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_psap}-user" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e log_level="INFO" -e policy_output="{BUCKET_PSAP}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") +os.system( + f"""podman run --rm --name cloud-governance --net="host" -e account="perf-scale" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf_scale}-user" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e policy_output="{BUCKET_PERF_SCALE}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/azure/daily/cost_reports/Jenkinsfile b/jenkins/clouds/azure/daily/cost_reports/Jenkinsfile index ba5b1032..fcef37c6 100644 --- a/jenkins/clouds/azure/daily/cost_reports/Jenkinsfile +++ b/jenkins/clouds/azure/daily/cost_reports/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AZURE_CLIENT_SECRET = credentials('cloud-governance-azure-client-secret') AZURE_TENANT_ID = credentials('cloud-governance-azure-tenant-id') AZURE_ACCOUNT_ID = credentials('cloud-governance-azure-account-id') @@ -27,7 +28,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Upload ElasticSearch') { @@ -37,7 +38,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/azure/daily/cost_reports/run_policies.py b/jenkins/clouds/azure/daily/cost_reports/run_policies.py index 5d393f62..cd6db748 100644 --- a/jenkins/clouds/azure/daily/cost_reports/run_policies.py +++ b/jenkins/clouds/azure/daily/cost_reports/run_policies.py @@ -1,5 +1,3 @@ - - import os AZURE_CLIENT_SECRET = os.environ['AZURE_CLIENT_SECRET'] @@ -10,15 +8,21 @@ COST_SPREADSHEET_ID = os.environ['COST_SPREADSHEET_ID'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] AZURE_ACCOUNT_ID = os.environ['AZURE_ACCOUNT_ID'] +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') print('Running the Azure cost billing reports') input_vars_to_container = [{'account': 'perf-scale-azure', 'AZURE_CLIENT_ID': AZURE_CLIENT_ID, - 'AZURE_TENANT_ID': AZURE_TENANT_ID, 'AZURE_CLIENT_SECRET': AZURE_CLIENT_SECRET, 'AZURE_ACCOUNT_ID': AZURE_ACCOUNT_ID}] + 'AZURE_TENANT_ID': AZURE_TENANT_ID, 'AZURE_CLIENT_SECRET': AZURE_CLIENT_SECRET, + 'AZURE_ACCOUNT_ID': AZURE_ACCOUNT_ID}] -common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': 'cloud-governance-clouds-billing-reports', 'log_level': 'INFO', 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS} +common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': 'cloud-governance-clouds-billing-reports', + 'log_level': 'INFO', 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS} combine_vars = lambda item: f'{item[0]}="{item[1]}"' common_envs = list(map(combine_vars, common_input_vars.items())) for input_vars in input_vars_to_container: envs = list(map(combine_vars, input_vars.items())) - os.system(f"""podman run --rm --name cloud-governance -e policy="cost_billing_reports" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" quay.io/ebattat/cloud-governance:latest""") - os.system(f"""podman run --rm --name cloud-governance -e COST_CENTER_OWNER="Shai" -e policy="cost_billing_reports" -e TOTAL_ACCOUNTS="True" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance -e policy="cost_billing_reports" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + os.system( + f"""podman run --rm --name cloud-governance -e COST_CENTER_OWNER="Shai" -e policy="cost_billing_reports" -e TOTAL_ACCOUNTS="True" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/azure/daily/policies/Jenkinsfile b/jenkins/clouds/azure/daily/policies/Jenkinsfile index 75e19438..1513978c 100644 --- a/jenkins/clouds/azure/daily/policies/Jenkinsfile +++ b/jenkins/clouds/azure/daily/policies/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') POLICIES_IN_ACTION = '[]' AZURE_CLIENT_SECRET = credentials('cloud-governance-azure-client-secret') AZURE_TENANT_ID = credentials('cloud-governance-azure-tenant-id') @@ -26,7 +27,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Azure Policies') { @@ -48,7 +49,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/azure/daily/policies/run_policies.py b/jenkins/clouds/azure/daily/policies/run_policies.py index b437fbda..6971dce7 100644 --- a/jenkins/clouds/azure/daily/policies/run_policies.py +++ b/jenkins/clouds/azure/daily/policies/run_policies.py @@ -12,7 +12,8 @@ LOGS = os.environ.get('LOGS', 'logs') ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] -CLOUD_GOVERNANCE = "quay.io/ebattat/cloud-governance:latest" +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') def get_policies(file_type: str = '.py', exclude_policies: list = None): @@ -57,7 +58,7 @@ def run_cmd(cmd: str): def get_container_cmd(env_dict: dict): env_list = ' '.join(list(map(lambda item: f'-e {item[0]}="{item[1]}"', env_dict.items()))) container_name = "cloud-governance" - container_run_cmd = f"""podman run --rm --name "{container_name}" --net="host" {env_list} {CLOUD_GOVERNANCE}""" + container_run_cmd = f"""podman run --rm --name "{container_name}" --net="host" {env_list} {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""" return container_run_cmd @@ -66,7 +67,6 @@ def get_container_cmd(env_dict: dict): policies_in_action = literal_eval(policies_in_action) policies_not_action = list(set(available_policies) - set(policies_in_action)) - container_env_dict = { "AZURE_CLIENT_SECRET": AZURE_CLIENT_SECRET, "AZURE_TENANT_ID": AZURE_TENANT_ID, diff --git a/jenkins/clouds/gcp/daily/cost_reports/Jenkinsfile b/jenkins/clouds/gcp/daily/cost_reports/Jenkinsfile index e638348b..5f2a4691 100644 --- a/jenkins/clouds/gcp/daily/cost_reports/Jenkinsfile +++ b/jenkins/clouds/gcp/daily/cost_reports/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') GCP_DATABASE_NAME = credentials('cloud-governance-gcp-database-name') GCP_DATABASE_TABLE_NAME = credentials('cloud-governance-gcp-database-table-name') ES_HOST = credentials('cloud-governance-es-host') @@ -25,7 +26,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Upload ElasticSearch') { @@ -35,7 +36,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/gcp/daily/cost_reports/run_reports.py b/jenkins/clouds/gcp/daily/cost_reports/run_reports.py index e89d1647..8dc5671f 100644 --- a/jenkins/clouds/gcp/daily/cost_reports/run_reports.py +++ b/jenkins/clouds/gcp/daily/cost_reports/run_reports.py @@ -1,5 +1,3 @@ - - import os GCP_DATABASE_NAME = os.environ['GCP_DATABASE_NAME'] @@ -8,6 +6,8 @@ ES_PORT = os.environ['ES_PORT'] COST_SPREADSHEET_ID = os.environ['COST_SPREADSHEET_ID'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') print('Running the GCP cost billing reports') @@ -18,4 +18,5 @@ combine_vars = lambda item: f'{item[0]}="{item[1]}"' common_envs = list(map(combine_vars, common_input_vars.items())) -os.system(f"""podman run --rm --name cloud-governance -e policy="cost_billing_reports" -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance -e policy="cost_billing_reports" -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/ibm/daily/cost_billings/Jenkinsfile b/jenkins/clouds/ibm/daily/cost_billings/Jenkinsfile index 8ef6354c..2ce2d85a 100644 --- a/jenkins/clouds/ibm/daily/cost_billings/Jenkinsfile +++ b/jenkins/clouds/ibm/daily/cost_billings/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') IBM_API_USERNAME_PERFORMANCE_SCALE = credentials('cloud-governance-ibm-api-username-performance-scale') @@ -35,7 +36,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -45,7 +46,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/ibm/daily/cost_billings/run_ibm_cost_reports.py b/jenkins/clouds/ibm/daily/cost_billings/run_ibm_cost_reports.py index 3c5ac56c..b0f8df18 100644 --- a/jenkins/clouds/ibm/daily/cost_billings/run_ibm_cost_reports.py +++ b/jenkins/clouds/ibm/daily/cost_billings/run_ibm_cost_reports.py @@ -1,7 +1,5 @@ - import os - LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] IBM_API_KEY_PERFORMANCE_SCALE = os.environ['IBM_API_KEY_PERFORMANCE_SCALE'] @@ -20,14 +18,24 @@ ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] LOGS = os.environ.get('LOGS', 'logs') +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') print('Run IBM Cost Forecast, Budget upload monthly') es_index = 'cloud-governance-clouds-billing-reports' -key_list = [{"account": "performance-scale", "IBM_API_USERNAME": IBM_API_USERNAME_PERFORMANCE_SCALE, "IBM_API_KEY": IBM_API_KEY_PERFORMANCE_SCALE, "USAGE_REPORTS_APIKEY": USAGE_REPORTS_APIKEY_PERFORMANCE_SCALE, "IBM_ACCOUNT_ID": IBM_ACCOUNT_ID_PERFORMANCE_SCALE}, - {"account": "interop-ibm-ci", "IBM_API_USERNAME": IBM_API_USERNAME_INTEROP_IBM_CI, "IBM_API_KEY": IBM_API_KEY_INTEROP_IBM_CI, "USAGE_REPORTS_APIKEY": USAGE_REPORTS_APIKEY_INTEROP_IBM_CI, "IBM_ACCOUNT_ID": IBM_ACCOUNT_ID_INTEROP_IBM_CI}, - {"account": "certification-ce", "IBM_API_USERNAME": IBM_API_USERNAME_CERTIFICATION_CE, "IBM_API_KEY": IBM_API_KEY_CERTIFICATION_CE, "USAGE_REPORTS_APIKEY": USAGE_REPORTS_APIKEY_CERTIFICATION_CE, "IBM_ACCOUNT_ID": IBM_ACCOUNT_ID_CERTIFICATION_CE}] +key_list = [{"account": "performance-scale", "IBM_API_USERNAME": IBM_API_USERNAME_PERFORMANCE_SCALE, + "IBM_API_KEY": IBM_API_KEY_PERFORMANCE_SCALE, + "USAGE_REPORTS_APIKEY": USAGE_REPORTS_APIKEY_PERFORMANCE_SCALE, + "IBM_ACCOUNT_ID": IBM_ACCOUNT_ID_PERFORMANCE_SCALE}, + {"account": "interop-ibm-ci", "IBM_API_USERNAME": IBM_API_USERNAME_INTEROP_IBM_CI, + "IBM_API_KEY": IBM_API_KEY_INTEROP_IBM_CI, "USAGE_REPORTS_APIKEY": USAGE_REPORTS_APIKEY_INTEROP_IBM_CI, + "IBM_ACCOUNT_ID": IBM_ACCOUNT_ID_INTEROP_IBM_CI}, + {"account": "certification-ce", "IBM_API_USERNAME": IBM_API_USERNAME_CERTIFICATION_CE, + "IBM_API_KEY": IBM_API_KEY_CERTIFICATION_CE, "USAGE_REPORTS_APIKEY": USAGE_REPORTS_APIKEY_CERTIFICATION_CE, + "IBM_ACCOUNT_ID": IBM_ACCOUNT_ID_CERTIFICATION_CE}] for keys in key_list: - os.system(f"""podman run --rm --name cloud-governance -e account="{keys.get('account')}" -e COST_CENTER_OWNER="Shai" -e policy="cost_billing_reports" -e es_index="{es_index}" -e es_port="{ES_PORT}" -e es_host="{ES_HOST}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e "IBM_API_USERNAME"="{keys.get('IBM_API_USERNAME')}" -e IBM_API_KEY="{keys.get('IBM_API_KEY')}" -e USAGE_REPORTS_APIKEY="{keys.get('USAGE_REPORTS_APIKEY')}" -e IBM_ACCOUNT_ID="{keys.get('IBM_ACCOUNT_ID')}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance -e account="{keys.get('account')}" -e COST_CENTER_OWNER="Shai" -e policy="cost_billing_reports" -e es_index="{es_index}" -e es_port="{ES_PORT}" -e es_host="{ES_HOST}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e "IBM_API_USERNAME"="{keys.get('IBM_API_USERNAME')}" -e IBM_API_KEY="{keys.get('IBM_API_KEY')}" -e USAGE_REPORTS_APIKEY="{keys.get('USAGE_REPORTS_APIKEY')}" -e IBM_ACCOUNT_ID="{keys.get('IBM_ACCOUNT_ID')}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile b/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile index c10551d2..066210bf 100644 --- a/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile +++ b/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile @@ -10,6 +10,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') IBM_API_USERNAME = credentials('cloud-governance-ibm-api-username-performance-scale') IBM_API_KEY = credentials('cloud-governance-ibm-api-key-performance-scale') AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') @@ -26,7 +27,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -36,7 +37,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/ibm/hourly/tagging/tagging.py b/jenkins/clouds/ibm/hourly/tagging/tagging.py index 5b8e89ef..5b261d85 100644 --- a/jenkins/clouds/ibm/hourly/tagging/tagging.py +++ b/jenkins/clouds/ibm/hourly/tagging/tagging.py @@ -1,16 +1,17 @@ - import os - LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] IBM_API_KEY = os.environ['IBM_API_KEY'] IBM_API_USERNAME = os.environ['IBM_API_USERNAME'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] LOGS = os.environ.get('LOGS', 'logs') - +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') print('Run IBM tagging on baremetal, vm') -os.system(f"""podman run --rm --name cloud-governance -e account="IBM-PERF" -e policy="tag_baremetal" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") -os.system(f"""podman run --rm --name cloud-governance -e account="IBM-PERF" -e policy="tag_vm" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance -e account="IBM-PERF" -e policy="tag_baremetal" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") +os.system( + f"""podman run --rm --name cloud-governance -e account="IBM-PERF" -e policy="tag_vm" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/ibm/monthly/cost_invoice/Jenkinsfile b/jenkins/clouds/ibm/monthly/cost_invoice/Jenkinsfile index 3b1bbfc6..56a762dd 100644 --- a/jenkins/clouds/ibm/monthly/cost_invoice/Jenkinsfile +++ b/jenkins/clouds/ibm/monthly/cost_invoice/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') IBM_API_USERNAME = credentials('cloud-governance-ibm-api-username-performance-scale') @@ -25,7 +26,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -35,7 +36,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/ibm/monthly/cost_invoice/ibm_invoice_to_es.py b/jenkins/clouds/ibm/monthly/cost_invoice/ibm_invoice_to_es.py index 30d954b1..0135da94 100644 --- a/jenkins/clouds/ibm/monthly/cost_invoice/ibm_invoice_to_es.py +++ b/jenkins/clouds/ibm/monthly/cost_invoice/ibm_invoice_to_es.py @@ -1,7 +1,5 @@ - import os - LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] IBM_API_KEY = os.environ['IBM_API_KEY'] @@ -10,9 +8,12 @@ ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] LOGS = os.environ.get('LOGS', 'logs') +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') print('Run IBM Cost Invoice upload monthly') es_index = 'cloud-governance-ibm-invoice-cost' -os.system(f"""podman run --rm --name cloud-governance -e account="IBM-PERF" -e policy="ibm_cost_report" -e es_index="{es_index}" -e es_port="{ES_PORT}" -e es_host="{ES_HOST}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance -e account="IBM-PERF" -e policy="ibm_cost_report" -e es_index="{es_index}" -e es_port="{ES_PORT}" -e es_host="{ES_HOST}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/ibm/weekly/cost_over_usage/Jenkinsfile b/jenkins/clouds/ibm/weekly/cost_over_usage/Jenkinsfile index 27e7aea9..c1ec324c 100644 --- a/jenkins/clouds/ibm/weekly/cost_over_usage/Jenkinsfile +++ b/jenkins/clouds/ibm/weekly/cost_over_usage/Jenkinsfile @@ -10,6 +10,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') USAGE_REPORTS_APIKEY = credentials('cloud-governance-ibm-service-id-performance-scale') IBM_ACCOUNT_ID = credentials('cloud-governance-ibm-account-id-performance-scale') contact1 = "ebattat@redhat.com" @@ -23,7 +24,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -33,7 +34,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/clouds/ibm/weekly/cost_over_usage/run_policies.py b/jenkins/clouds/ibm/weekly/cost_over_usage/run_policies.py index b4b86534..056237a6 100644 --- a/jenkins/clouds/ibm/weekly/cost_over_usage/run_policies.py +++ b/jenkins/clouds/ibm/weekly/cost_over_usage/run_policies.py @@ -6,5 +6,8 @@ CC_MAIL = ['athiruma@redhat.com', 'ebattat@redhat.com'] USAGE_REPORTS_AUTHTYPE = 'iam' MAXIMUM_THRESHOLD = 1000 +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') -os.system(f"""podman run --rm --name cloud-governance --net=host -e policy="ibm_cost_over_usage" -e account="IBM-PERF" -e IBM_ACCOUNT_ID="{IBM_ACCOUNT_ID}" -e to_mail="{TO_MAIL}" -e cc_mail="{CC_MAIL}" -e USAGE_REPORTS_APIKEY="{USAGE_REPORTS_APIKEY}" -e USAGE_REPORTS_AUTHTYPE="{USAGE_REPORTS_AUTHTYPE}" -e MAXIMUM_THRESHOLD="{MAXIMUM_THRESHOLD}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance --net=host -e policy="ibm_cost_over_usage" -e account="IBM-PERF" -e IBM_ACCOUNT_ID="{IBM_ACCOUNT_ID}" -e to_mail="{TO_MAIL}" -e cc_mail="{CC_MAIL}" -e USAGE_REPORTS_APIKEY="{USAGE_REPORTS_APIKEY}" -e USAGE_REPORTS_AUTHTYPE="{USAGE_REPORTS_AUTHTYPE}" -e MAXIMUM_THRESHOLD="{MAXIMUM_THRESHOLD}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/poc/haim/appeng/PolicyJenkinsfileDaily b/jenkins/poc/haim/appeng/PolicyJenkinsfileDaily index 84036029..04de5201 100644 --- a/jenkins/poc/haim/appeng/PolicyJenkinsfileDaily +++ b/jenkins/poc/haim/appeng/PolicyJenkinsfileDaily @@ -30,7 +30,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' } } stage('Run Policies the Cost Policies') { @@ -45,7 +45,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/poc/haim/appeng/TaggingJenkinsfileHourly b/jenkins/poc/haim/appeng/TaggingJenkinsfileHourly index ed68d2f5..aad0d7f7 100644 --- a/jenkins/poc/haim/appeng/TaggingJenkinsfileHourly +++ b/jenkins/poc/haim/appeng/TaggingJenkinsfileHourly @@ -25,7 +25,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -35,7 +35,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/poc/haim/common/run_cost_policies.py b/jenkins/poc/haim/common/run_cost_policies.py index a8e4440f..8fd826f6 100644 --- a/jenkins/poc/haim/common/run_cost_policies.py +++ b/jenkins/poc/haim/common/run_cost_policies.py @@ -7,11 +7,11 @@ ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] - cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email', 'Environment', 'User:Spot'] cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost granularity = 'DAILY' # DAILY/MONTHLY/HOURLY cost_explorer_index = 'cloud-governance-haim-cost-explorer-global-index' os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """) -os.system(f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/common/run_policies.py b/jenkins/poc/haim/common/run_policies.py index 41d0f765..cc39b2d1 100644 --- a/jenkins/poc/haim/common/run_policies.py +++ b/jenkins/poc/haim/common/run_policies.py @@ -1,8 +1,8 @@ - import os from ast import literal_eval -policies_in_action = ['ebs_unattached', 'ip_unattached', 'zombie_snapshots', 'unused_nat_gateway', 's3_inactive', 'empty_roles'] +policies_in_action = ['ebs_unattached', 'ip_unattached', 'zombie_snapshots', 'unused_nat_gateway', 's3_inactive', + 'empty_roles'] policies_not_in_action = ['ec2_stop', 'instance_idle', 'zombie_cluster_resource'] access_key = os.environ['access_key'] @@ -26,18 +26,23 @@ os.system(f"echo Polices list: {policies_not_in_action}") for region in regions: for policy in policies_not_in_action: - os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") if policy == 'zombie_cluster_resource': - os.system(f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="{account_name}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_doc_type="{es_doc_type}" -e bucket="{s3_bucket}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="{account_name}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_doc_type="{es_doc_type}" -e bucket="{s3_bucket}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") os.system('echo "Running the CloudGovernance policies with dry_run=no" ') os.system(f"echo Polices list: {policies_in_action}") for region in regions: for policy in policies_in_action: if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1': - os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") elif policy not in ('empty_roles', 's3_inactive'): - os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") os.system(f"""echo "Running the tag_iam_user" """) -os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/common/run_tagging.py b/jenkins/poc/haim/common/run_tagging.py index 8cfb017e..d185d1ec 100644 --- a/jenkins/poc/haim/common/run_tagging.py +++ b/jenkins/poc/haim/common/run_tagging.py @@ -1,19 +1,17 @@ - import os - access_key = os.environ['access_key'] secret_key = os.environ['secret_key'] account_name = os.environ['account_name'] LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] - LOGS = os.environ.get('LOGS', 'logs') mandatory_tags_appeng = {'Budget': account_name} - os.system(f"""echo "Running the tag_resources" """) -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', + 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] for region in regions: - os.system(f"""podman run --rm --name cloud-governance-poc-haim -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/daily/Jenkinsfile b/jenkins/poc/haim/daily/Jenkinsfile index 616cb281..52c458e4 100644 --- a/jenkins/poc/haim/daily/Jenkinsfile +++ b/jenkins/poc/haim/daily/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' } } stage('Run Policies for haim poc') { @@ -44,7 +44,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/poc/haim/daily/es_upload.py b/jenkins/poc/haim/daily/es_upload.py index 65987c78..58bd0d22 100644 --- a/jenkins/poc/haim/daily/es_upload.py +++ b/jenkins/poc/haim/daily/es_upload.py @@ -1,7 +1,5 @@ - import os - AWS_ACCESS_KEY_ID_APPENG = os.environ['AWS_ACCESS_KEY_ID_APPENG'] AWS_SECRET_ACCESS_KEY_APPENG = os.environ['AWS_SECRET_ACCESS_KEY_APPENG'] LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] @@ -17,7 +15,9 @@ def get_policies(type: str = None): @return: list of policies name """ policies = [] - policies_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))), 'cloud_governance', 'policy', 'aws') + policies_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))), + 'cloud_governance', 'policy', 'aws') for (dirpath, dirnames, filenames) in os.walk(policies_path): for filename in filenames: if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')): @@ -28,7 +28,8 @@ def get_policies(type: str = None): return policies -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', + 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] os.system('echo "Upload data to ElasticSearch - ec2 index"') @@ -40,6 +41,8 @@ def get_policies(type: str = None): for policy in policies: if policy in ('empty_roles', 's3_inactive'): if region == 'us-east-1': - os.system(f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="APPENG" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index}" -e es_doc_type="{es_doc_type}" -e bucket="{BUCKET_APPENG}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="APPENG" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index}" -e es_doc_type="{es_doc_type}" -e bucket="{BUCKET_APPENG}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") else: - os.system(f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="APPENG" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index}" -e es_doc_type="{es_doc_type}" -e bucket="{BUCKET_APPENG}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="APPENG" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index}" -e es_doc_type="{es_doc_type}" -e bucket="{BUCKET_APPENG}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/daily/run_policies.py b/jenkins/poc/haim/daily/run_policies.py index c6aedc4a..2bc22c6a 100644 --- a/jenkins/poc/haim/daily/run_policies.py +++ b/jenkins/poc/haim/daily/run_policies.py @@ -1,7 +1,5 @@ - import os - AWS_ACCESS_KEY_ID_APPENG = os.environ['AWS_ACCESS_KEY_ID_APPENG'] AWS_SECRET_ACCESS_KEY_APPENG = os.environ['AWS_SECRET_ACCESS_KEY_APPENG'] LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] @@ -17,7 +15,9 @@ def get_policies(type: str = None): @return: list of policies name """ policies = [] - policies_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))), 'cloud_governance', 'policy', 'aws') + policies_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))), + 'cloud_governance', 'policy', 'aws') for (dirpath, dirnames, filenames) in os.walk(policies_path): for filename in filenames: if not filename.startswith('__') and (filename.endswith('.yml') or filename.endswith('.py')): @@ -28,26 +28,30 @@ def get_policies(type: str = None): return policies -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', + 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] policies = get_policies() -not_action_policies = ['cost_explorer', 'cost_over_usage', 'monthly_report', 'cost_billing_reports', 'cost_explorer_payer_billings'] +not_action_policies = ['cost_explorer', 'cost_over_usage', 'monthly_report', 'cost_billing_reports', + 'cost_explorer_payer_billings'] run_policies = list(set(policies) - set(not_action_policies)) run_policies.sort() - os.system(f"""echo Running the cloud_governance policies: {run_policies}""") os.system(f"""echo "Running the CloudGovernance policies" """) for region in regions: for policy in run_policies: if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1': - os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="APPENG" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{BUCKET_APPENG}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="APPENG" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{BUCKET_APPENG}/{LOGS}/{region}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") else: - os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="APPENG" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{BUCKET_APPENG}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") - + os.system( + f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="APPENG" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{BUCKET_APPENG}/{LOGS}/{region}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") -cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email', 'Environment', 'User:Spot'] +cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', + 'Email', 'Environment', 'User:Spot'] cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost granularity = 'DAILY' # DAILY/MONTHLY/HOURLY cost_explorer_index = 'cloud-governance-haim-cost-explorer-global-index' os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """) -os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="appeng" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="appeng" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/poc/haim/hourly/Jenkinsfile b/jenkins/poc/haim/hourly/Jenkinsfile index 696a043a..0715995b 100644 --- a/jenkins/poc/haim/hourly/Jenkinsfile +++ b/jenkins/poc/haim/hourly/Jenkinsfile @@ -26,7 +26,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -36,7 +36,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/cloud-governance/cloud-governance 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/poc/haim/hourly/run_policies.py b/jenkins/poc/haim/hourly/run_policies.py index 173483ff..9fdbf21e 100644 --- a/jenkins/poc/haim/hourly/run_policies.py +++ b/jenkins/poc/haim/hourly/run_policies.py @@ -1,23 +1,22 @@ - import os - AWS_ACCESS_KEY_ID_APPENG = os.environ['AWS_ACCESS_KEY_ID_APPENG'] AWS_SECRET_ACCESS_KEY_APPENG = os.environ['AWS_SECRET_ACCESS_KEY_APPENG'] LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] - LOGS = os.environ.get('LOGS', 'logs') mandatory_tags_appeng = {'Budget': 'APPENG'} os.system(f"""echo "Running the tag_iam_user" """) -os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="APPENG" -e -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") - +os.system( + f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="APPENG" -e -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/cloud-governance/cloud-governance:latest""") os.system(f"""echo "Running the tag_resources" """) -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', + 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1'] for region in regions: - os.system(f"""podman run --rm --name cloud-governance-poc-haim -e account="APPENG" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim -e account="APPENG" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_APPENG}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_APPENG}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/cloud-governance/cloud-governance:latest""") diff --git a/jenkins/tenant/aws/common/run_cost_policies.py b/jenkins/tenant/aws/common/run_cost_policies.py index 5bee76e1..7af0e079 100644 --- a/jenkins/tenant/aws/common/run_cost_policies.py +++ b/jenkins/tenant/aws/common/run_cost_policies.py @@ -6,7 +6,8 @@ account_name = os.environ['account_name'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] - +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance') cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email', 'Environment', 'User:Spot', 'cluster_id'] @@ -14,4 +15,5 @@ granularity = 'DAILY' # DAILY/MONTHLY/HOURLY cost_explorer_index = 'cloud-governance-haim-cost-explorer-global-index' os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """) -os.system(f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system( + f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/tenant/aws/common/run_policies.py b/jenkins/tenant/aws/common/run_policies.py index 2459b58d..ba152029 100644 --- a/jenkins/tenant/aws/common/run_policies.py +++ b/jenkins/tenant/aws/common/run_policies.py @@ -23,8 +23,13 @@ def get_policies(file_type: str = '.py', exclude_policies: list = None): exclude_policies = ['cost_explorer', 'optimize_resources_report', 'monthly_report', 'cost_over_usage', - 'skipped_resources', 'cost_explorer_payer_billings', 'cost_billing_reports', 'spot_savings_analysis'] + 'skipped_resources', 'cost_explorer_payer_billings', 'cost_billing_reports', + 'spot_savings_analysis'] available_policies = get_policies(exclude_policies=exclude_policies) +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance') + + # available_policies: Run policies in dry_run="yes" mode @@ -44,7 +49,7 @@ def get_container_cmd(env_dict: dict): env_list = ' '.join(list(map(create_container_envs, env_dict.items()))) container_name = "cloud-governance-poc-haim" container_run_cmd = f""" -podman run --rm --name "{container_name}" --net="host" {env_list} quay.io/ebattat/cloud-governance:latest +podman run --rm --name "{container_name}" --net="host" {env_list} {QUAY_CLOUD_GOVERNANCE_REPOSITORY} """ return container_run_cmd @@ -62,14 +67,12 @@ def get_container_cmd(env_dict: dict): GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] ADMIN_MAIL_LIST = os.environ.get('ADMIN_MAIL_LIST', '') -CLOUD_GOVERNANCE_IMAGE = "quay.io/ebattat/cloud-governance:latest" policies_in_action = os.environ.get('POLICIES_IN_ACTION', []) if isinstance(policies_in_action, str): policies_in_action = literal_eval(policies_in_action) policies_not_action = list(set(available_policies) - set(policies_in_action)) - regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1'] @@ -110,11 +113,11 @@ def run_policies(policies: list, dry_run: str = 'yes'): run_cmd(f"echo Polices list: {policies_in_action}") run_policies(policies=policies_in_action, dry_run='no') - run_cmd(f"""echo "Running the tag_iam_user" """) -run_cmd(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") - +run_cmd( + f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") # Run the AggMail -run_cmd(f"""podman run --rm --name cloud-governance-haim --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" -e ALERT_DRY_RUN="{ALERT_DRY_RUN}" {CLOUD_GOVERNANCE_IMAGE}""") +run_cmd( + f"""podman run --rm --name cloud-governance-haim --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" -e ALERT_DRY_RUN="{ALERT_DRY_RUN}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/tenant/aws/common/run_tagging.py b/jenkins/tenant/aws/common/run_tagging.py index 64668f0f..07288e7a 100644 --- a/jenkins/tenant/aws/common/run_tagging.py +++ b/jenkins/tenant/aws/common/run_tagging.py @@ -1,19 +1,20 @@ - import os - access_key = os.environ['access_key'] secret_key = os.environ['secret_key'] account_name = os.environ['account_name'] LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] - +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance') LOGS = os.environ.get('LOGS', 'logs') mandatory_tags_appeng = {'Budget': account_name} - os.system(f"""echo "Running the tag_resources" """) -regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1'] +regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', + 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-northeast-1', 'ca-central-1', 'sa-east-1', + 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1'] for region in regions: - os.system(f"""podman run --rm --name cloud-governance-poc-haim -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""") + os.system( + f"""podman run --rm --name cloud-governance-poc-haim -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/tenant/aws/default/PolicyJenkinsfileDaily b/jenkins/tenant/aws/default/PolicyJenkinsfileDaily index 08c94606..c9a0cc5a 100644 --- a/jenkins/tenant/aws/default/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/default/PolicyJenkinsfileDaily @@ -11,6 +11,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') @@ -27,7 +28,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Policies the Cost Policies') { @@ -62,7 +63,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/tenant/aws/default/TaggingJenkinsfileHourly b/jenkins/tenant/aws/default/TaggingJenkinsfileHourly index 1cdae307..cc275ad1 100644 --- a/jenkins/tenant/aws/default/TaggingJenkinsfileHourly +++ b/jenkins/tenant/aws/default/TaggingJenkinsfileHourly @@ -11,6 +11,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') account_name = "appeng" contact1 = "ebattat@redhat.com" @@ -24,7 +25,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -44,7 +45,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily index 0871b435..9f74328e 100644 --- a/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily @@ -26,6 +26,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') @@ -49,7 +50,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Cost Policies') { @@ -87,7 +88,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/tenant/aws/ecoeng_01/TaggingJenkinsfileHourly b/jenkins/tenant/aws/ecoeng_01/TaggingJenkinsfileHourly index aa34d4ef..49c494a1 100644 --- a/jenkins/tenant/aws/ecoeng_01/TaggingJenkinsfileHourly +++ b/jenkins/tenant/aws/ecoeng_01/TaggingJenkinsfileHourly @@ -13,6 +13,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') contact1 = "ebattat@redhat.com" contact2 = "athiruma@redhat.com" @@ -27,7 +28,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -47,7 +48,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily index dfafd950..ca412a95 100644 --- a/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily @@ -14,6 +14,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') @@ -37,7 +38,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Cost Policies') { @@ -73,7 +74,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/tenant/aws/ecoeng_02/TaggingJenkinsfileHourly b/jenkins/tenant/aws/ecoeng_02/TaggingJenkinsfileHourly index 00cfb147..ee02e98d 100644 --- a/jenkins/tenant/aws/ecoeng_02/TaggingJenkinsfileHourly +++ b/jenkins/tenant/aws/ecoeng_02/TaggingJenkinsfileHourly @@ -11,6 +11,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') contact1 = "ebattat@redhat.com" contact2 = "athiruma@redhat.com" @@ -25,7 +26,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -45,7 +46,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily index d06060d6..fd767f60 100644 --- a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily @@ -11,6 +11,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') @@ -34,7 +35,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Cost Policies') { @@ -70,7 +71,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/jenkins/tenant/aws/ecoeng_03/TaggingJenkinsfileHourly b/jenkins/tenant/aws/ecoeng_03/TaggingJenkinsfileHourly index db65c594..e18af33a 100644 --- a/jenkins/tenant/aws/ecoeng_03/TaggingJenkinsfileHourly +++ b/jenkins/tenant/aws/ecoeng_03/TaggingJenkinsfileHourly @@ -11,6 +11,7 @@ pipeline { } } environment { + QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') contact1 = "ebattat@redhat.com" contact2 = "athiruma@redhat.com" @@ -25,7 +26,7 @@ pipeline { } stage('Initial Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' } } stage('Run Tagging Cluster & Non-Cluster') { @@ -45,7 +46,7 @@ pipeline { } stage('Finalize Cleanup') { steps { - sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi''' + sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' deleteDir() } } diff --git a/pod_yaml/cloud-governance.yaml b/pod_yaml/cloud-governance.yaml index 387d2925..909887f5 100644 --- a/pod_yaml/cloud-governance.yaml +++ b/pod_yaml/cloud-governance.yaml @@ -10,37 +10,37 @@ spec: name: cloud-governance spec: containers: - - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: cloud-governance-secret - key: aws_access_key_id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: cloud-governance-secret - key: aws_secret_access_key - - name: AWS_DEFAULT_REGION - valueFrom: - configMapKeyRef: - name: cloud-governance-config - key: cloud-governance.region - - name: policy - valueFrom: - configMapKeyRef: - name: cloud-governance-config - key: cloud-governance.policy - - name: dry_run - valueFrom: - configMapKeyRef: - name: cloud-governance-config - key: cloud-governance.dry_run - - name: log_level - valueFrom: - configMapKeyRef: - name: cloud-governance-config - key: cloud-governance.log_level - image: quay.io/ebattat/cloud-governance - name: cloud-governance + - env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: cloud-governance-secret + key: aws_access_key_id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: cloud-governance-secret + key: aws_secret_access_key + - name: AWS_DEFAULT_REGION + valueFrom: + configMapKeyRef: + name: cloud-governance-config + key: cloud-governance.region + - name: policy + valueFrom: + configMapKeyRef: + name: cloud-governance-config + key: cloud-governance.policy + - name: dry_run + valueFrom: + configMapKeyRef: + name: cloud-governance-config + key: cloud-governance.dry_run + - name: log_level + valueFrom: + configMapKeyRef: + name: cloud-governance-config + key: cloud-governance.log_level + image: quay.io/cloud-governance/cloud-governance + name: cloud-governance restartPolicy: OnFailure diff --git a/pod_yaml/upload_quayio b/pod_yaml/upload_quayio index 49681f86..c7490bda 100644 --- a/pod_yaml/upload_quayio +++ b/pod_yaml/upload_quayio @@ -2,5 +2,5 @@ # twine upload dist/* # sudo podman build --layers=false -t cloud-governance:v1.0.190 . # sudo podman login quay.io -# sudo podman push localhost/cloud-governance:v1.0.105 quay.io/ebattat/cloud-governance -# sudo podman pull quay.io/ebattat/cloud-governance +# sudo podman push localhost/cloud-governance:v1.0.105 quay.io/cloud-governance/cloud-governance +# sudo podman pull quay.io/cloud-governance/cloud-governance