(feat) Add XFCC authenticated scaling engine endpoint (#3320)

* Adds cf_Server config for scalingengine * Adds xfcc cf endpoint support to scaling engine * Remove debug println * Update jobs/scalingengine/spec Co-authored-by: Silvestre Zabala <[email protected]> * Update jobs/scalingengine/spec Co-authored-by: Silvestre Zabala <[email protected]> * Remove pending test * Removes subrouter in api server --------- Co-authored-by: Silvestre Zabala <[email protected]>
cloudfoundry · Nov 27, 2024 · de4bb15 · de4bb15
1 parent 72ccecf
commit de4bb15
Show file tree

Hide file tree

Showing 30 changed files with 887 additions and 585 deletions.
diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh
@@ -1,65 +1,55 @@
-#! /usr/bin/env bash
+#!/usr/bin/env bash
 # shellcheck disable=SC2086,SC2034,SC2155
 set -euo pipefail
 
-script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
 source "${script_dir}/vars.source.sh"
+source "${script_dir}/common.sh"
 
 deployment_manifest="${autoscaler_dir}/templates/app-autoscaler.yml"
 bosh_deploy_opts="${BOSH_DEPLOY_OPTS:-""}"
+BOSH_DEPLOY_VARS="${BOSH_DEPLOY_VARS:-""}"
 bosh_upload_release_opts="${BOSH_UPLOAD_RELEASE_OPTS:-""}"
 bosh_upload_stemcell_opts="${BOSH_UPLOAD_STEMCELL_OPTS:-""}"
-ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\
-  ${autoscaler_dir}/operations/instance-identity-cert-from-cf.yml\
-  ${autoscaler_dir}/operations/add-postgres-variables.yml\
-  ${autoscaler_dir}/operations/connect_to_postgres_with_certs.yml\
-  ${autoscaler_dir}/operations/enable-nats-tls.yml\
-  ${autoscaler_dir}/operations/add-extra-plan.yml\
-  ${autoscaler_dir}/operations/set-release-version.yml\
-  ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\
-  ${autoscaler_dir}/operations/enable-scheduler-logging.yml"}
-
-
-## if mtar_deployment_enabled, then apply the use-cf operator file
-if [[  "${enable_mtar}"  == "true" ]]; then
-  echo "Deploying with mtar enabled"
-  ops_files+=" ${autoscaler_dir}/operations/use-cf-services.yml"
-fi
+
+ops_files=${OPS_FILES:-$(cat <<EOF
+${autoscaler_dir}/operations/add-releases.yml
+${autoscaler_dir}/operations/instance-identity-cert-from-cf.yml
+${autoscaler_dir}/operations/add-postgres-variables.yml
+${autoscaler_dir}/operations/connect_to_postgres_with_certs.yml
+${autoscaler_dir}/operations/enable-nats-tls.yml
+${autoscaler_dir}/operations/add-extra-plan.yml
+${autoscaler_dir}/operations/set-release-version.yml
+${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml
+${autoscaler_dir}/operations/enable-scheduler-logging.yml
+EOF
+)}
 
 case "${cpu_upper_threshold}" in
-  "100")
-  # default
-  ;;
-  "200")
-  ops_files+=" ${autoscaler_dir}/operations/cpu_upper_threshold_200.yml"
-  ;;
-  "400")
-  ops_files+=" ${autoscaler_dir}/operations/cpu_upper_threshold_400.yml"
-  ;;
-  *)
-  echo "No Ops file for cpu_upper_threshold of ${cpu_upper_threshold}"
-  exit 1
-  ;;
+  "100") ;;
+  "200") ops_files+=" ${autoscaler_dir}/operations/cpu_upper_threshold_200.yml" ;;
+  "400") ops_files+=" ${autoscaler_dir}/operations/cpu_upper_threshold_400.yml" ;;
+  *) echo "No Ops file for cpu_upper_threshold of ${cpu_upper_threshold}"; exit 1 ;;
 esac
 
 CURRENT_COMMIT_HASH=$(cd "${autoscaler_dir}"; git log -1 --pretty=format:"%H")
 bosh_release_version=${RELEASE_VERSION:-${CURRENT_COMMIT_HASH}-${deployment_name}}
 
 pushd "${bbl_state_path}" > /dev/null
-  eval "$(bbl print-env)"
+eval "$(bbl print-env)"
 popd > /dev/null
 
-function setup_autoscaler_uaac(){
+function setup_autoscaler_uaac() {
   local uaac_authorities="cloud_controller.read,cloud_controller.admin,uaa.resource,routing.routes.write,routing.routes.read,routing.router_groups.read"
   local autoscaler_secret="autoscaler_client_secret"
   local uaa_client_secret=$(credhub get -n /bosh-autoscaler/cf/uaa_admin_client_secret --quiet)
+
   uaac target "https://uaa.${system_domain}" --skip-ssl-validation > /dev/null
   uaac token client get admin -s "${uaa_client_secret}" > /dev/null
 
   if uaac client get autoscaler_client_id >/dev/null; then
     step "updating autoscaler uaac client"
-    uaac client update "autoscaler_client_id" \
-      --authorities "$uaac_authorities" > /dev/null
+    uaac client update "autoscaler_client_id" --authorities "$uaac_authorities" > /dev/null
   else
     step "creating autoscaler uaac client"
     uaac client add "autoscaler_client_id" \
@@ -68,59 +58,58 @@ function setup_autoscaler_uaac(){
       --secret "$autoscaler_secret" > /dev/null
   fi
 }
-function get_postgres_external_port(){
-	if [ -z "${PR_NUMBER}" ]; then
-		echo "5432"
-	else
-		echo "${PR_NUMBER}"
-	fi
+
+function get_postgres_external_port() {
+  [[ -z "${PR_NUMBER}" ]] && echo "5432" || echo "${PR_NUMBER}"
 }
 
-function create_manifest(){
-  # Set the local tmp_dir depending on if we run on github-actions or not, see:
-  # https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+function create_manifest() {
   local tmp_dir
-  local perform_as_gh_action
+  local perform_as_gh_action="${GITHUB_ACTIONS:-false}"
 
-  perform_as_gh_action="${GITHUB_ACTIONS:-false}"
-  if "${perform_as_gh_action}" != 'false'
-  then
+  if "${perform_as_gh_action}" != 'false'; then
     tmp_dir="${RUNNER_TEMP}"
-  else # local system
+  else
     tmp_dir="$(pwd)/dev_releases"
     mkdir -p "${tmp_dir}"
   fi
 
-  # on MacOS mktemp does not know the --tmpdir option
   tmp_manifest_file="$(mktemp "${tmp_dir}/${deployment_name}.bosh-manifest.yaml.XXX")"
+  credhub interpolate -f "${autoscaler_dir}/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl" > /tmp/autoscaler-secrets.yml
 
+	add_variable "deployment_name" "${deployment_name}"
+	add_variable "system_domain" "${system_domain}"
+	add_variable "app_autoscaler_version" "${bosh_release_version}"
+	add_variable "cf_client_id" "autoscaler_client_id"
+	add_variable "cf_client_secret" "autoscaler_client_secret"
+	add_variable "postgres_external_port" "$(get_postgres_external_port)"
 
-  credhub interpolate -f "${autoscaler_dir}/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl" > /tmp/autoscaler-secrets.yml
 
-  bosh -n -d "${deployment_name}" \
-      interpolate "${deployment_manifest}" \
-      ${OPS_FILES_TO_USE} \
-      ${bosh_deploy_opts} \
-      -v system_domain="${system_domain}" \
-      -v deployment_name="${deployment_name}" \
-      -v app_autoscaler_version="${bosh_release_version}" \
-      -v cf_client_id=autoscaler_client_id \
-      -v cf_client_secret=autoscaler_client_secret \
-      -v postgres_external_port="$(get_postgres_external_port)"\
-			--vars-file=/tmp/autoscaler-secrets.yml \
-      -v skip_ssl_validation=true \
-      > "${tmp_manifest_file}"
+	bosh_deploy_vars=""
 
+	# add deployment name
+  bosh -n -d "${deployment_name}" interpolate "${deployment_manifest}" ${OPS_FILES_TO_USE} \
+    ${bosh_deploy_opts} ${BOSH_DEPLOY_VARS} \
+    --vars-file=/tmp/autoscaler-secrets.yml -v skip_ssl_validation=true > "${tmp_manifest_file}"
 
+  if [[ -z "${debug}" || "${debug}" = "false" ]]; then
     # shellcheck disable=SC2064
-  if [ -z "${debug}" ] || [ "${debug}" = "false" ] ; then  trap "rm ${tmp_manifest_file}" EXIT ; fi
+    trap "rm ${tmp_manifest_file}" EXIT
+  fi
 }
 
-function check_ops_files(){
+add_variable() {
+	local variable_name=$1
+	local variable_value=$2
+	BOSH_DEPLOY_VARS="${BOSH_DEPLOY_VARS} -v ${variable_name}=${variable_value}"
+}
+
+function check_ops_files() {
   step "Using Ops files: '${ops_files}'"
+
   OPS_FILES_TO_USE=""
   for OPS_FILE in ${ops_files}; do
-    if [ -f "${OPS_FILE}" ]; then
+    if [[ -f "${OPS_FILE}" ]]; then
       OPS_FILES_TO_USE="${OPS_FILES_TO_USE} -o ${OPS_FILE}"
     else
       echo "ERROR: could not find ops file ${OPS_FILE} in ${PWD}"
@@ -130,45 +119,34 @@ function check_ops_files(){
 }
 
 function deploy() {
-  # Try to silence Prometheus but do not fail deployment if there's an error
-# ${script_dir}/silence_prometheus_alert.sh "BOSHJobEphemeralDiskPredictWillFill" || true
-#  ${script_dir}/silence_prometheus_alert.sh "BOSHJobProcessUnhealthy" || true
-#  ${script_dir}/silence_prometheus_alert.sh "BOSHJobUnhealthy" || true
-
   create_manifest
-
-  log "creating Bosh deployment '${deployment_name}' with version '${bosh_release_version}' in system domain '${system_domain}'   "
+  log "creating Bosh deployment '${deployment_name}' with version '${bosh_release_version}' in system domain '${system_domain}'"
   debug "tmp_manifest_file=${tmp_manifest_file}"
   step "Using Ops files: '${OPS_FILES_TO_USE}'"
   step "Deploy options: '${bosh_deploy_opts}'"
   bosh -n -d "${deployment_name}" deploy "${tmp_manifest_file}"
-	postgres_ip="$(bosh curl "/deployments/${deployment_name}/vms" | jq '. | .[] | select(.job == "postgres") | .ips[0]' -r)"
-	credhub set -n "/bosh-autoscaler/${deployment_name}/postgres_ip" -t value -v "${postgres_ip}"
-
+  postgres_ip="$(bosh curl "/deployments/${deployment_name}/vms" | jq '. | .[] | select(.job == "postgres") | .ips[0]' -r)"
+  credhub set -n "/bosh-autoscaler/${deployment_name}/postgres_ip" -t value -v "${postgres_ip}"
 }
 
 function find_or_upload_stemcell() {
-  # Determine if we need to upload a stemcell at this point.
-  stemcell_os=$(yq eval '.stemcells[] | select(.alias == "default").os' ${deployment_manifest})
-  stemcell_version=$(yq eval '.stemcells[] | select(.alias == "default").version' ${deployment_manifest})
+  local stemcell_os stemcell_version stemcell_name
+  stemcell_os=$(yq eval '.stemcells[] | select(.alias == "default").os' "${deployment_manifest}")
+  stemcell_version=$(yq eval '.stemcells[] | select(.alias == "default").version' "${deployment_manifest}")
   stemcell_name="bosh-google-kvm-${stemcell_os}-go_agent"
 
   if ! bosh stemcells | grep "${stemcell_name}" >/dev/null; then
-    URL="https://bosh.io/d/stemcells/${stemcell_name}"
-    if [ "${stemcell_version}" != "latest" ]; then
-      URL="${URL}?v=${stemcell_version}"
-    fi
+    local URL="https://bosh.io/d/stemcells/${stemcell_name}"
+    [[ "${stemcell_version}" != "latest" ]] && URL="${URL}?v=${stemcell_version}"
     wget "${URL}" -O stemcell.tgz
     bosh -n upload-stemcell $bosh_upload_stemcell_opts stemcell.tgz
   fi
 }
 
 function find_or_upload_release() {
   if ! bosh releases | grep -E "${bosh_release_version}[*]*\s" > /dev/null; then
-
-    local -r release_desc_file="dev_releases/app-autoscaler/app-autoscaler-${bosh_release_version}.yml"
-    if [ ! -f "${release_desc_file}" ]
-    then
+    local release_desc_file="dev_releases/app-autoscaler/app-autoscaler-${bosh_release_version}.yml"
+    if [[ ! -f "${release_desc_file}" ]]; then
       echo "Creating Release with bosh version ${bosh_release_version}"
       bosh create-release --force --version="${bosh_release_version}"
     else
@@ -185,11 +163,26 @@ function find_or_upload_release() {
   fi
 }
 
-log "Deploying autoscaler '${bosh_release_version}' with name '${deployment_name}' "
+function pre_deploy() {
+	if [[ "${enable_mtar}" == "true" ]]; then
+		ops_files+=" ${autoscaler_dir}/operations/use-cf-services.yml"
+		cf_login
+
+		local autoscaler_cf_server_xfcc_valid_org_guid=$(cf org ${AUTOSCALER_ORG} --guid)
+		local autoscaler_cf_server_xfcc_valid_space_guid=$(cf space ${AUTOSCALER_SPACE} --guid)
+
+		add_variable "autoscaler_cf_server_xfcc_valid_org_guid" "${autoscaler_cf_server_xfcc_valid_org_guid}"
+		add_variable "autoscaler_cf_server_xfcc_valid_space_guid" "${autoscaler_cf_server_xfcc_valid_space_guid}"
+	fi
+}
+
+log "Deploying autoscaler '${bosh_release_version}' with name '${deployment_name}'"
 setup_autoscaler_uaac
 pushd "${autoscaler_dir}" > /dev/null
-  check_ops_files
-  find_or_upload_stemcell
-  find_or_upload_release
-  deploy
+pre_deploy
+check_ops_files
+find_or_upload_stemcell
+find_or_upload_release
+deploy
 popd > /dev/null
+
diff --git a/jobs/scalingengine/spec b/jobs/scalingengine/spec
@@ -169,6 +169,17 @@ properties:
   autoscaler.scalingengine.server_key:
     description: "PEM-encoded server key"
 
+  autoscaler.cf_server.port:
+    description: "the listening port of cf xfcc endpoint"
+    default: 8080
+
+  autoscaler.cf_server.xfcc.valid_org_guid:
+    description: allowed org guid for xfcc endpoint
+    default: ''
+
+  autoscaler.cf_server.xfcc.valid_space_guid:
+    description: allowed space guid for xfcc endpoint
+    default: ''
 
   autoscaler.scalingengine.health.port:
     description: "the listening port of health endpoint"

diff --git a/jobs/scalingengine/templates/scalingengine.yml.erb b/jobs/scalingengine/templates/scalingengine.yml.erb
@@ -51,6 +51,11 @@ cf:
   idle_connection_timeout_ms: <%= p("autoscaler.cf.idle_connection_timeout_ms") %>
   max_idle_conns_per_host_ms: <%= p("autoscaler.cf.max_idle_conns_per_host_ms") %>
 
+cf_server:
+  port: <%= p("autoscaler.cf_server.port") %>
+  xfcc:
+    valid_org_guid: <%= p("autoscaler.cf_server.xfcc.valid_org_guid") %>
+    valid_space_guid: <%= p("autoscaler.cf_server.xfcc.valid_space_guid") %>
 
 server:
   port:  <%= p("autoscaler.scalingengine.server.port") %>

diff --git a/operations/use-cf-services.yml b/operations/use-cf-services.yml
@@ -50,3 +50,30 @@
 
 - type: remove
   path: /instance_groups/name=metricsforwarder
+
+
+## SCALINGENGINE - Enable cf Server to receive calls from api running on cf --
+- type: replace
+  path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scalingengine/cf_server?/xfcc?/valid_org_guid?
+  value: ((autoscaler_cf_server_xfcc_valid_org_guid))
+
+- type: replace
+  path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scalingengine/cf_server?/xfcc?/valid_space_guid?
+  value: ((autoscaler_cf_server_xfcc_valid_space_guid))
+
+
+- type: replace
+  path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scalingengine/cf_server?/port?
+  value: &scalingEngineCfPort 6205
+
+- type: replace
+  path: /instance_groups/name=postgres/jobs/name=route_registrar/properties/route_registrar/routes/-
+  value:
+    name: ((deployment_name))-cf-scalingengine
+    registration_interval: 20s
+    port: *scalingEngineCfPort
+    tags:
+      component: autoscaler_cf_scalingengine
+    uris:
+      - ((deployment_name))-cf-scalingengine.((system_domain))
+
diff --git a/packages/scalingengine/spec b/packages/scalingengine/spec
@@ -14,6 +14,7 @@ files:
 - autoscaler/db/sqldb/* # gosub
 - autoscaler/healthendpoint/* # gosub
 - autoscaler/helpers/* # gosub
+- autoscaler/helpers/auth/* # gosub
 - autoscaler/helpers/handlers/* # gosub
 - autoscaler/metricsforwarder/server/common/* # gosub
 - autoscaler/models/* # gosub

diff --git a/spec/jobs/scalingengine/scalingengine_spec.rb b/spec/jobs/scalingengine/scalingengine_spec.rb
@@ -39,6 +39,23 @@
       end
     end
 
+    context "cf server" do
+      it "includes default port for cf server" do
+        expect(rendered_template["cf_server"]["port"]).to eq(8080)
+      end
+
+      it "defaults xfcc valid org and space " do
+        properties["autoscaler"]["cf_server"] = {}
+        properties["autoscaler"]["cf_server"]["xfcc"] = {
+          "valid_org_guid" => "some-valid-org-guid",
+          "valid_space_guid" => "some-valid-space-guid"
+        }
+
+        expect(rendered_template["cf_server"]["xfcc"]["valid_org_guid"]).to eq(properties["autoscaler"]["cf_server"]["xfcc"]["valid_org_guid"])
+        expect(rendered_template["cf_server"]["xfcc"]["valid_space_guid"]).to eq(properties["autoscaler"]["cf_server"]["xfcc"]["valid_space_guid"])
+      end
+    end
+
     context "uses tls" do
       context "policy_db" do
         it "includes the ca, cert and key in url when configured" do

diff --git a/src/autoscaler/api/brokerserver/broker_server.go b/src/autoscaler/api/brokerserver/broker_server.go
@@ -63,7 +63,7 @@ func (am *AuthMiddleware) authenticate(r *http.Request) bool {
 }
 
 type BrokerServer interface {
-	GetServer() (ifrit.Runner, error)
+	CreateServer() (ifrit.Runner, error)
 	GetRouter() (*chi.Mux, error)
 }
 
@@ -89,7 +89,7 @@ func NewBrokerServer(logger lager.Logger, conf *config.Config, bindingDB db.Bind
 	}
 }
 
-func (s *brokerServer) GetServer() (ifrit.Runner, error) {
+func (s *brokerServer) CreateServer() (ifrit.Runner, error) {
 	router, err := s.GetRouter()
 	if err != nil {
 		return nil, err

diff --git a/src/autoscaler/api/brokerserver/broker_server_suite_test.go b/src/autoscaler/api/brokerserver/broker_server_suite_test.go
@@ -145,7 +145,7 @@ var _ = BeforeSuite(func() {
 	fakeCredentials := &fakes.FakeCredentials{}
 	httpStatusCollector := &fakes.FakeHTTPStatusCollector{}
 	bs := brokerserver.NewBrokerServer(lager.NewLogger("test"), conf, fakeBindingDB, fakePolicyDB, httpStatusCollector, nil, fakeCredentials)
-	httpServer, err := bs.GetServer()
+	httpServer, err := bs.CreateServer()
 	Expect(err).NotTo(HaveOccurred())
 
 	serverUrl, err = url.Parse("http://localhost:" + strconv.Itoa(port))