From 699b4d73962100bad6ea1f8f61c83f672bac8470 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 23 Oct 2024 15:57:31 +0200 Subject: [PATCH 1/7] Refactor deployment script and add autoscaler-secrets.yml template - Add a new autoscaler-secrets.yml.tpl file for managing autoscaler secrets - Streamline secret injection by using `credhub interpolate` with the new secrets template - Remove direct `credhub get` calls for secrets in the deploy-autoscaler.sh script - Comment out Prometheus alert silencing commands in the deploy function --- .../scripts/autoscaler-secrets.yml.tpl | 13 ++++++++++ ci/autoscaler/scripts/deploy-autoscaler.sh | 24 ++++++------------- 2 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 ci/autoscaler/scripts/autoscaler-secrets.yml.tpl diff --git a/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl b/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl new file mode 100644 index 0000000000..2c25984c38 --- /dev/null +++ b/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl @@ -0,0 +1,13 @@ +--- +admin_password: ((/bosh-autoscaler/cf/cf_admin_password)) +routing_api_tls_client_cert: ((/bosh-autoscaler/cf/routing_api_tls_client.certificate)) +routing_api_ca_certs: ((/bosh-autoscaler/cf/router_ssl.ca)) +routing_api_client_secret: ((/bosh-autoscaler/cf/uaa_clients_routing_api_client_secret)) +routing_api_tls_client_private_key: ((/bosh-autoscaler/cf/routing_api_tls_client.private_key)) +routing_api_server_ca_cert: ((/bosh-autoscaler/cf/router_ssl.ca)) +log_cache_syslog_tls_ca: ((/bosh-autoscaler/cf/log_cache_syslog_tls.ca)) +syslog_agent_log_cache_tls_certificate: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.certificate)) +syslog_agent_log_cache_tls_key: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.private_key)) +metricscollector_ca_cert: ((/bosh-autoscaler/cf/log_cache.ca)) +metricscollector_client_cert: ((/bosh-autoscaler/cf/log_cache.certificate)) +metricscollector_client_key: ((/bosh-autoscaler/cf/log_cache.private_key)) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 7cdbc0c6c8..24d61fa28e 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -20,7 +20,6 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} - case "${cpu_upper_threshold}" in "100") # default @@ -89,6 +88,9 @@ function create_manifest(){ # on MacOS mktemp does not know the --tmpdir option tmp_manifest_file="$(mktemp "${tmp_dir}/${deployment_name}.bosh-manifest.yaml.XXX")" + + credhub interpolate -f "${autoscaler_dir}/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl" > /tmp/autoscaler-secrets.yml + bosh -n -d "${deployment_name}" \ interpolate "${deployment_manifest}" \ ${OPS_FILES_TO_USE} \ @@ -96,22 +98,10 @@ function create_manifest(){ -v system_domain="${system_domain}" \ -v deployment_name="${deployment_name}" \ -v app_autoscaler_version="${bosh_release_version}" \ - -v admin_password="$(credhub get -n /bosh-autoscaler/cf/cf_admin_password -q)"\ - -v routing_api_ca_certs="$(credhub get -n /bosh-autoscaler/cf/router_ssl --key ca --quiet)"\ - -v routing_api_client_secret="$(credhub get -n /bosh-autoscaler/cf/uaa_clients_routing_api_client_secret --quiet)"\ - -v routing_api_tls_client_cert="$(credhub get -n /bosh-autoscaler/cf/routing_api_tls_client --key certificate --quiet)"\ - -v routing_api_tls_client_private_key="$(credhub get -n /bosh-autoscaler/cf/routing_api_tls_client --key private_key --quiet)"\ - -v routing_api_server_ca_cert="$(credhub get -n /bosh-autoscaler/cf/router_ssl --key ca --quiet)"\ -v cf_client_id=autoscaler_client_id \ -v cf_client_secret=autoscaler_client_secret \ - -v log_cache_syslog_tls_ca="$(credhub get -n /bosh-autoscaler/cf/log_cache_syslog_tls --key ca --quiet)"\ - -v syslog_agent_log_cache_tls_certificate="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet)"\ - -v syslog_agent_log_cache_tls_key="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)"\ - -v metricscollector_ca_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key ca --quiet)"\ - -v metricscollector_client_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key certificate --quiet)"\ - -v metricsforwarder_host="${metricsforwarder_host}"\ -v postgres_external_port="$(get_postgres_external_port)"\ - -v metricscollector_client_key="$(credhub get -n /bosh-autoscaler/cf/log_cache --key private_key --quiet)"\ + --vars-file=/tmp/autoscaler-secrets.yml \ -v skip_ssl_validation=true \ > "${tmp_manifest_file}" @@ -134,9 +124,9 @@ function check_ops_files(){ function deploy() { # Try to silence Prometheus but do not fail deployment if there's an error - ${script_dir}/silence_prometheus_alert.sh "BOSHJobEphemeralDiskPredictWillFill" || true - ${script_dir}/silence_prometheus_alert.sh "BOSHJobProcessUnhealthy" || true - ${script_dir}/silence_prometheus_alert.sh "BOSHJobUnhealthy" || true +# ${script_dir}/silence_prometheus_alert.sh "BOSHJobEphemeralDiskPredictWillFill" || true +# ${script_dir}/silence_prometheus_alert.sh "BOSHJobProcessUnhealthy" || true +# ${script_dir}/silence_prometheus_alert.sh "BOSHJobUnhealthy" || true create_manifest From c376f1ba511cd3d0c3d51e092c84fd7cd03b9d70 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 23 Oct 2024 16:26:00 +0200 Subject: [PATCH 2/7] Add mtar deployment option to autoscaler script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Implement conditional inclusion of use-cf-services.yml based on enable_mtar flag in deploy-autoscaler.sh • Introduce ENABLE_MTAR environment variable with default false in vars.source.sh --- ci/autoscaler/scripts/deploy-autoscaler.sh | 6 ++++++ ci/autoscaler/scripts/vars.source.sh | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 24d61fa28e..f6f0e8e9de 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -20,6 +20,12 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} +## if mtar_deployment_enabled, then apply the use-cf operator file +if [[ "${enable_mtar}" == "true" ]]; then + echo "Deploying with mtar enabled" + ops_files+=" ${autoscaler_dir}/operations/use-cf-services.yml" +fi + case "${cpu_upper_threshold}" in "100") # default diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index b7b925a2ac..accb81d798 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -123,3 +123,7 @@ debug "PERFORMANCE_TEARDOWN: ${PERFORMANCE_TEARDOWN}" export CPU_UPPER_THRESHOLD=${CPU_UPPER_THRESHOLD:-100} debug "CPU_UPPER_THRESHOLD: ${CPU_UPPER_THRESHOLD}" cpu_upper_threshold=${CPU_UPPER_THRESHOLD} + +export ENABLE_MTAR=${ENABLE_MTAR:-false} +debug "ENABLE_MTAR: ${ENABLE_MTAR}" +enable_mtar=${ENABLE_MTAR} From c6d079afb49e89ce33bcf37e1b7273be6132f78b Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 23 Oct 2024 17:58:48 +0200 Subject: [PATCH 3/7] Add metricsforwarder_host variable to deploy-autoscaler.sh manifest creation function --- ci/autoscaler/scripts/deploy-autoscaler.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index f6f0e8e9de..4dd4f8aa4d 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -107,6 +107,7 @@ function create_manifest(){ -v cf_client_id=autoscaler_client_id \ -v cf_client_secret=autoscaler_client_secret \ -v postgres_external_port="$(get_postgres_external_port)"\ + -v metricsforwarder_host="${metricsforwarder_host}"\ --vars-file=/tmp/autoscaler-secrets.yml \ -v skip_ssl_validation=true \ > "${tmp_manifest_file}" From ff3a39b613f4f0f6656900716569d9c380d7d22b Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 24 Oct 2024 11:20:54 +0200 Subject: [PATCH 4/7] Updated Makefile to use DEPLOYMENT_NAME for security group creation and binding in metricsforwarder --- src/autoscaler/metricsforwarder/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 1d3b9f9d95..e0d49443c1 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -5,5 +5,5 @@ set-security-group: $(eval ORG := $(shell cf target |grep "org\:" |cut -d':' -f2 | xargs)) $(eval SPACE := $(shell cf target |grep "space\:" |cut -d':' -f2 | xargs)) - cf create-security-group metricsforwarder $(MAKEFILE_DIR)/security-group.json - cf bind-security-group metricsforwarder $(ORG) + cf create-security-group $(DEPLOYMENT_NAME) $(MAKEFILE_DIR)/security-group.json + cf bind-security-group $(DEPLOYMENT_NAME) $(ORG) From fdf8ebb93496dea1bf065724f1d635f84233f099 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 24 Oct 2024 15:00:05 +0200 Subject: [PATCH 5/7] Add cf_target invocation to cf-login script --- ci/autoscaler/scripts/cf-login.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/autoscaler/scripts/cf-login.sh b/ci/autoscaler/scripts/cf-login.sh index a73986a56e..f1b5a73e54 100755 --- a/ci/autoscaler/scripts/cf-login.sh +++ b/ci/autoscaler/scripts/cf-login.sh @@ -6,5 +6,7 @@ source "${script_dir}/common.sh" bosh_login cf_login +cf_target "${AUTOSCALER_ORG}" "${AUTOSCALER_SPACE}" + cf autoscaling-api "https://autoscaler-${PR_NUMBER}.${SYSTEM_DOMAIN}" From 37dec348e47deb4cd758f704b7c92cfef95feddf Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 24 Oct 2024 15:24:50 +0200 Subject: [PATCH 6/7] Refactor secret management in build-extension-file.sh using credhub interpolate and yq MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Replace direct credhub secret retrieval with credhub interpolate to generate a secrets YAML file. • Utilize yq to extract secrets from the generated YAML file for environment variable assignment. • Streamline the process of setting environment variables for metrics forwarder and policy DB credentials, as well as syslog client certificates. --- src/autoscaler/build-extension-file.sh | 29 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/autoscaler/build-extension-file.sh b/src/autoscaler/build-extension-file.sh index a1adb01176..bb0e9b9737 100755 --- a/src/autoscaler/build-extension-file.sh +++ b/src/autoscaler/build-extension-file.sh @@ -19,17 +19,30 @@ export SYSTEM_DOMAIN="autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" export POSTGRES_ADDRESS="${DEPLOYMENT_NAME}-postgres.tcp.${SYSTEM_DOMAIN}" export POSTGRES_EXTERNAL_PORT="${PR_NUMBER:-5432}" -export METRICSFORWARDER_HEALTH_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler_metricsforwarder_health_password --quiet)" +cat << EOF > /tmp/extension-file-secrets.yml.tpl +metricsforwarder_health_password: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler_metricsforwarder_health_password)) +policy_db_password: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/database_password)) +policy_db_server_ca: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server.ca)) +policy_db_client_cert: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server.certificate)) +policy_db_client_key: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server.private_key)) +syslog_client_ca: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.ca)) +syslog_client_cert: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.certificate)) +syslog_client_key: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.private_key)) +EOF + +credhub interpolate -f "/tmp/extension-file-secrets.yml.tpl" > /tmp/mtar-secrets.yml + export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}" +export METRICSFORWARDER_HEALTH_PASSWORD="$(yq ".metricsforwarder_health_password" /tmp/mtar-secrets.yml)" -export POLICY_DB_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/database_password --quiet)" -export POLICY_DB_SERVER_CA="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key ca --quiet )" -export POLICY_DB_CLIENT_CERT="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key certificate --quiet)" -export POLICY_DB_CLIENT_KEY="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key private_key --quiet)" +export POLICY_DB_PASSWORD="$(yq ".policy_db_password" /tmp/mtar-secrets.yml)" +export POLICY_DB_SERVER_CA="$(yq ".policy_db_server_ca" /tmp/mtar-secrets.yml)" +export POLICY_DB_CLIENT_CERT="$(yq ".policy_db_client_cert" /tmp/mtar-secrets.yml)" +export POLICY_DB_CLIENT_KEY="$(yq ".policy_db_client_key" /tmp/mtar-secrets.yml)" -export SYSLOG_CLIENT_CA="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key ca --quiet)" -export SYSLOG_CLIENT_CERT="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet)" -export SYSLOG_CLIENT_KEY="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)" +export SYSLOG_CLIENT_CA="$(yq ".syslog_client_ca" /tmp/mtar-secrets.yml)" +export SYSLOG_CLIENT_CERT="$(yq ".syslog_client_cert" /tmp/mtar-secrets.yml)" +export SYSLOG_CLIENT_KEY="$(yq ".syslog_client_key" /tmp/mtar-secrets.yml)" cat < "${extension_file_path}" ID: development From 637937253c93934d7da0608de380b695f8be66bb Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 24 Oct 2024 15:25:28 +0200 Subject: [PATCH 7/7] Update metricsforwarder security group creation and binding in Makefile --- src/autoscaler/metricsforwarder/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index e0d49443c1..7061fbd267 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -5,5 +5,6 @@ set-security-group: $(eval ORG := $(shell cf target |grep "org\:" |cut -d':' -f2 | xargs)) $(eval SPACE := $(shell cf target |grep "space\:" |cut -d':' -f2 | xargs)) - cf create-security-group $(DEPLOYMENT_NAME) $(MAKEFILE_DIR)/security-group.json - cf bind-security-group $(DEPLOYMENT_NAME) $(ORG) + cf create-security-group metricsforwarder $(MAKEFILE_DIR)/security-group.json + cf update-security-group metricsforwarder $(MAKEFILE_DIR)/security-group.json + cf bind-security-group metricsforwarder $(ORG)