From 3ccd60511ef51e1bd413d7996e91101aec377e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Mac=C3=ADk?= Date: Mon, 30 Sep 2024 22:46:25 +0200 Subject: [PATCH] fix(RHIXP-3670): Improve framework to be more reliable at high scalability values (#90) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Pavel Macík --- Makefile | 4 +- ci-scripts/collect-results.sh | 60 ++++++++-- ci-scripts/prow-to-storage.sh | 30 ++--- ci-scripts/rhdh-setup/create_resource.sh | 113 +++++++++++------- ci-scripts/rhdh-setup/deploy.sh | 77 +++++++----- .../template/backstage/app-config.yaml | 6 +- .../backstage/olm/rbac-plugin-patch.yaml | 3 + ci-scripts/runs-to-csv.sh | 42 ++++++- 8 files changed, 228 insertions(+), 107 deletions(-) create mode 100644 ci-scripts/rhdh-setup/template/backstage/olm/rbac-plugin-patch.yaml diff --git a/Makefile b/Makefile index b452805..f843fcc 100644 --- a/Makefile +++ b/Makefile @@ -173,8 +173,8 @@ clean: test: $(TMP_DIR) $(ARTIFACT_DIR) echo $(SCENARIO)>$(TMP_DIR)/benchmark-scenario ifneq ($(shell test '$(AUTH_PROVIDER)' == 'keycloak' && echo 1 || echo 0),0) - $(eval key_pass := $(shell oc -n rhdh-performance get secret perf-test-secrets -o template --template='{{.data.keycloak_user_pass}}' | base64 -d)) - $(eval key_host := $(shell oc -n rhdh-performance get routes/keycloak -o template --template='{{.spec.host}}' )) + $(eval key_pass := $(shell oc -n $(RHDH_NAMESPACE) get secret perf-test-secrets -o template --template='{{.data.keycloak_user_pass}}' | base64 -d)) + $(eval key_host := $(shell oc -n $(RHDH_NAMESPACE) get routes/keycloak -o template --template='{{.spec.host}}' )) $(eval LOCUST_EXTRA_CMD := $(LOCUST_EXTRA_CMD) --keycloak-host $(key_host) --keycloak-password $(key_pass) ) ifneq ($(shell test $(USERS) -gt $(WORKERS) && echo 1 || echo 0),0) @echo "users greater than workers " diff --git a/ci-scripts/collect-results.sh b/ci-scripts/collect-results.sh index 16e9af2..93ccb51 100755 --- a/ci-scripts/collect-results.sh +++ b/ci-scripts/collect-results.sh @@ -65,6 +65,10 @@ try_gather_file "${TMP_DIR}/deploy-before" try_gather_file "${TMP_DIR}/deploy-after" try_gather_file "${TMP_DIR}/populate-before" try_gather_file "${TMP_DIR}/populate-after" +try_gather_file "${TMP_DIR}/populate-users-groups-before" +try_gather_file "${TMP_DIR}/populate-users-groups-after" +try_gather_file "${TMP_DIR}/populate-catalog-before" +try_gather_file "${TMP_DIR}/populate-catalog-after" try_gather_file "${TMP_DIR}/benchmark-before" try_gather_file "${TMP_DIR}/benchmark-after" try_gather_file "${TMP_DIR}/benchmark-scenario" @@ -93,11 +97,42 @@ set +u # shellcheck disable=SC1090,SC1091 source $PYTHON_VENV_DIR/bin/activate set -u + +timestamp_diff() { + started="$1" + ended="$2" + echo "$(date -d "$ended" +"%s.%N") - $(date -d "$started" +"%s.%N")" | bc +} + # populate phase if [ "$PRE_LOAD_DB" == "true" ]; then - mstart=$(date --utc --date "$(cat "${TMP_DIR}/populate-before")" --iso-8601=seconds) - mend=$(date --utc --date "$(cat "${TMP_DIR}/populate-after")" --iso-8601=seconds) + mstart=$(date --utc --date "$(cat "${ARTIFACT_DIR}/populate-before")" --iso-8601=seconds) + mend=$(date --utc --date "$(cat "${ARTIFACT_DIR}/populate-after")" --iso-8601=seconds) mhost=$(kubectl -n openshift-monitoring get route -l app.kubernetes.io/name=thanos-query -o json | jq --raw-output '.items[0].spec.host') + deploy_started=$(cat "${ARTIFACT_DIR}/deploy-before") + deploy_ended=$(cat "${ARTIFACT_DIR}/deploy-after") + populate_started=$(cat "${ARTIFACT_DIR}/populate-before") + populate_ended=$(cat "${ARTIFACT_DIR}/populate-after") + populate_users_groups_started=$(cat "${ARTIFACT_DIR}/populate-users-groups-before") + populate_users_groups_ended=$(cat "${ARTIFACT_DIR}/populate-users-groups-after") + populate_catalog_started=$(cat "${ARTIFACT_DIR}/populate-catalog-before") + populate_catalog_ended=$(cat "${ARTIFACT_DIR}/populate-catalog-after") + status_data.py \ + --status-data-file "$monitoring_collection_data" \ + --set \ + measurements.timings.deploy.started="$deploy_started" \ + measurements.timings.deploy.ended="$deploy_ended" \ + measurements.timings.deploy.duration="$(timestamp_diff "$deploy_started" "$deploy_ended")" \ + measurements.timings.populate.started="$populate_started" \ + measurements.timings.populate.ended="$populate_ended" \ + measurements.timings.populate.duration="$(timestamp_diff "$populate_started" "$populate_ended")" \ + measurements.timings.populate_users_groups.started="$populate_users_groups_started" \ + measurements.timings.populate_users_groups.ended="$populate_users_groups_ended" \ + measurements.timings.populate_users_groups.duration="$(timestamp_diff "$populate_users_groups_started" "$populate_users_groups_ended")" \ + measurements.timings.populate_catalog.started="$populate_catalog_started" \ + measurements.timings.populate_catalog.ended="$populate_catalog_ended" \ + measurements.timings.populate_catalog.duration="$(timestamp_diff "$populate_catalog_started" "$populate_catalog_ended")" \ + -d &>"$monitoring_collection_log" status_data.py \ --status-data-file "$monitoring_collection_data" \ --additional config/cluster_read_config.populate.yaml \ @@ -110,17 +145,20 @@ if [ "$PRE_LOAD_DB" == "true" ]; then -d &>>"$monitoring_collection_log" fi # test phase -mstart=$(date --utc --date "$(cat "${TMP_DIR}/benchmark-before")" --iso-8601=seconds) -mend=$(date --utc --date "$(cat "${TMP_DIR}/benchmark-after")" --iso-8601=seconds) +mstart=$(date --utc --date "$(cat "${ARTIFACT_DIR}/benchmark-before")" --iso-8601=seconds) +mend=$(date --utc --date "$(cat "${ARTIFACT_DIR}/benchmark-after")" --iso-8601=seconds) mhost=$(kubectl -n openshift-monitoring get route -l app.kubernetes.io/name=thanos-query -o json | jq --raw-output '.items[0].spec.host') -mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' "scenarios/$(cat "${TMP_DIR}/benchmark-scenario").py") +mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' "scenarios/$(cat "${ARTIFACT_DIR}/benchmark-scenario").py") +benchmark_started=$(cat "${ARTIFACT_DIR}/benchmark-before") +benchmark_ended=$(cat "${ARTIFACT_DIR}/benchmark-after") status_data.py \ --status-data-file "$monitoring_collection_data" \ --set \ - results.started="$(cat "${TMP_DIR}/benchmark-before")" \ - results.ended="$(cat "${TMP_DIR}/benchmark-after")" \ - name="RHDH load test $(cat "${TMP_DIR}/benchmark-scenario")" \ - metadata.scenario.name="$(cat "${TMP_DIR}/benchmark-scenario")" \ + measurements.timings.benchmark.started="$benchmark_started" \ + measurements.timings.benchmark.ended="$benchmark_ended" \ + measurements.timings.benchmark.duration="$(timestamp_diff "$benchmark_started" "$benchmark_ended")" \ + name="RHDH load test $(cat "${ARTIFACT_DIR}/benchmark-scenario")" \ + metadata.scenario.name="$(cat "${ARTIFACT_DIR}/benchmark-scenario")" \ metadata.scenario.version="$mversion" \ -d &>"$monitoring_collection_log" status_data.py \ @@ -134,10 +172,10 @@ status_data.py \ --prometheus-token "$($cli whoami -t)" \ -d &>>"$monitoring_collection_log" #Scenario specific metrics -if [ -f "scenarios/$(cat "${TMP_DIR}/benchmark-scenario").metrics.yaml" ]; then +if [ -f "scenarios/$(cat "${ARTIFACT_DIR}/benchmark-scenario").metrics.yaml" ]; then status_data.py \ --status-data-file "$monitoring_collection_data" \ - --additional "scenarios/$(cat "${TMP_DIR}/benchmark-scenario").metrics.yaml" \ + --additional "scenarios/$(cat "${ARTIFACT_DIR}/benchmark-scenario").metrics.yaml" \ --monitoring-start "$mstart" \ --monitoring-end "$mend" \ --monitoring-raw-data-dir "$monitoring_collection_dir" \ diff --git a/ci-scripts/prow-to-storage.sh b/ci-scripts/prow-to-storage.sh index d45a429..fe45895 100755 --- a/ci-scripts/prow-to-storage.sh +++ b/ci-scripts/prow-to-storage.sh @@ -21,7 +21,7 @@ if [ -z "$HORREUM_JHUTAR_PASSWORD" ]; then fi function _log() { - echo "$( date -Ins --utc ) $1 $2" >&1 + echo "$(date -Ins --utc) $1 $2" >&1 } function debug() { @@ -85,8 +85,8 @@ function check_json_string() { function check_result() { # Ensure benchmark.json have all the required fields => test finished local f="$1" - if jq -e '.results.ended == null' "$f" >/dev/null; then - error "File is missing .results.ended, skipping" + if jq -e '.measurements.timings.benchmark.ended == null' "$f" >/dev/null; then + error "File is missing .measurements.timings.benchmark.ended, skipping" return 1 fi if jq -e '.results | length == 0' "$f" >/dev/null; then @@ -101,7 +101,7 @@ function enritch_stuff() { local key="$2" local value="$3" local current_in_file - current_in_file=$( jq --raw-output "$key" "$f" ) + current_in_file=$(jq --raw-output "$key" "$f") if [[ "$current_in_file" == "None" ]]; then debug "Adding $key to JSON file" jq "$key = \"$value\"" "$f" >"$$.json" && mv -f "$$.json" "$f" @@ -120,9 +120,9 @@ function upload_es() { debug "Considering file for upload to ES" local current_doc_in_es current_count_in_es current_error_in_es - current_doc_in_es="$( curl --silent -X GET $ES_HOST/$ES_INDEX/_search -H 'Content-Type: application/json' -d '{"query":{"term":{"metadata.env.BUILD_ID.keyword":{"value":"'"$build_id"'"}}}}' )" - current_count_in_es="$( echo "$current_doc_in_es" | jq --raw-output .hits.total.value )" - current_error_in_es="$( echo "$current_doc_in_es" | jq --raw-output .error.type )" + current_doc_in_es="$(curl --silent -X GET $ES_HOST/$ES_INDEX/_search -H 'Content-Type: application/json' -d '{"query":{"term":{"metadata.env.BUILD_ID.keyword":{"value":"'"$build_id"'"}}}}')" + current_count_in_es="$(echo "$current_doc_in_es" | jq --raw-output .hits.total.value)" + current_error_in_es="$(echo "$current_doc_in_es" | jq --raw-output .error.type)" if [[ "$current_error_in_es" == "index_not_found_exception" ]]; then info "Index does not exist yet, going on" @@ -154,8 +154,8 @@ function upload_horreum() { local test_start test_end TOKEN test_id exists ids_list is_fail - test_start="$( format_date "$( jq --raw-output '.results.started | if . == "" then "-" else . end' "$f" )" )" - test_end="$( format_date "$( jq --raw-output '.results.ended | if . == "" then "-" else . end' "$f" )" )" + test_start="$(format_date "$(jq --raw-output '.measurements.timings.benchmark.started | if . == "" then "-" else . end' "$f")")" + test_end="$(format_date "$(jq --raw-output '.measurements.timings.benchmark.ended | if . == "" then "-" else . end' "$f")")" if [ -z "$test_start" ] || [ -z "$test_end" ] || [ "$test_start" == "null" ] || [ "$test_end" == "null" ]; then error "We need start ($test_start) and end ($test_end) time in the JSON we are supposed to upload" @@ -164,11 +164,11 @@ function upload_horreum() { debug "Considering file upload to Horreum: start: $test_start, end: $test_end, $test_matcher: $build_id" - TOKEN=$( curl -s $HORREUM_KEYCLOAK_HOST/realms/horreum/protocol/openid-connect/token -d "username=jhutar@redhat.com" -d "password=$HORREUM_JHUTAR_PASSWORD" -d "grant_type=password" -d "client_id=horreum-ui" | jq --raw-output .access_token ) + TOKEN=$(curl -s $HORREUM_KEYCLOAK_HOST/realms/horreum/protocol/openid-connect/token -d "username=jhutar@redhat.com" -d "password=$HORREUM_JHUTAR_PASSWORD" -d "grant_type=password" -d "client_id=horreum-ui" | jq --raw-output .access_token) - test_id=$( curl --silent --get -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" "$HORREUM_HOST/api/test/byName/$test_name" | jq --raw-output .id ) + test_id=$(curl --silent --get -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" "$HORREUM_HOST/api/test/byName/$test_name" | jq --raw-output .id) - exists=$( curl --silent --get -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" "$HORREUM_HOST/api/dataset/list/$test_id" --data-urlencode "filter={\"$test_matcher\":\"$build_id\"}" | jq --raw-output '.datasets | length' ) + exists=$(curl --silent --get -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" "$HORREUM_HOST/api/dataset/list/$test_id" --data-urlencode "filter={\"$test_matcher\":\"$build_id\"}" | jq --raw-output '.datasets | length') if [[ $exists -gt 0 ]]; then info "Test result ($test_matcher=$build_id) found in Horreum ($exists), skipping upload" @@ -184,7 +184,7 @@ function upload_horreum() { echo info "Getting pass/fail for file from Horreum" - ids_list=$( curl --silent "https://horreum.corp.redhat.com/api/alerting/variables?test=$test_id" | jq -r '.[] | .id' ) + ids_list=$(curl --silent "https://horreum.corp.redhat.com/api/alerting/variables?test=$test_id" | jq -r '.[] | .id') is_fail=0 for i in $ids_list; do data='{ @@ -198,7 +198,7 @@ function upload_horreum() { } }' - count=$( curl --silent -H "Content-Type: application/json" "https://horreum.corp.redhat.com/api/changes/annotations" -d "$data" | jq -r '. | length' ) + count=$(curl --silent -H "Content-Type: application/json" "https://horreum.corp.redhat.com/api/changes/annotations" -d "$data" | jq -r '. | length') if [ "$count" -gt 0 ]; then is_fail=1 enritch_stuff "$f" ".result" "FAIL" @@ -226,7 +226,7 @@ for job in "mvp-cpt"; do enritch_stuff "$out" ".\"\$schema\"" "$HORREUM_TEST_SCHEMA" upload_horreum "$out" "$HORREUM_TEST_NAME" ".metadata.env.BUILD_ID" "$i" upload_es "$out" "$i" - (( counter++ )) || true + ((counter++)) || true done done diff --git a/ci-scripts/rhdh-setup/create_resource.sh b/ci-scripts/rhdh-setup/create_resource.sh index 34cd992..69eca9d 100755 --- a/ci-scripts/rhdh-setup/create_resource.sh +++ b/ci-scripts/rhdh-setup/create_resource.sh @@ -58,7 +58,7 @@ backstage_url() { } create_per_grp() { - echo "[INFO][$(date --utc -Ins)] Creating entity YAML files" + log_info "Creating entity YAML files" varname=$2 obj_count=${!varname} if [[ -z ${!varname} ]]; then @@ -89,7 +89,7 @@ create_per_grp() { } clone_and_upload() { - echo "[INFO][$(date --utc -Ins)] Uploading entities to GitHub" + log_info "Uploading entities to GitHub" git_str="${GITHUB_USER}:${GITHUB_TOKEN}@github.com" base_name=$(basename "$GITHUB_REPO") git_dir=$TMP_DIR/${base_name} @@ -113,22 +113,32 @@ clone_and_upload() { for filename in "${files[@]}"; do e_count=$(yq eval '.metadata.name | capture(".*-(?P[0-9]+)").value' "$filename" | tail -n 1) upload_url="${GITHUB_REPO%.*}/blob/${tmp_branch}/$(basename "$filename")" - echo "Uploading entities from $upload_url" + log_info "Uploading entities from $upload_url" ACCESS_TOKEN=$(get_token "rhdh") curl -k "$(backstage_url)/api/catalog/locations" --cookie "$COOKIE" --cookie-jar "$COOKIE" -X POST -H 'Accept-Encoding: gzip, deflate, br' -H 'Authorization: Bearer '"$ACCESS_TOKEN" -H 'Content-Type: application/json' --data-raw '{"type":"url","target":"'"${upload_url}"'"}' - timeout_timestamp=$(date -d "600 seconds" "+%s") + timeout=600 + timeout_timestamp=$(date -d "$timeout seconds" "+%s") + last_count=-1 while true; do if [ "$(date "+%s")" -gt "$timeout_timestamp" ]; then - echo "ERROR: Timeout waiting on entity count" + log_error "Timeout waiting on entity count" exit 1 else ACCESS_TOKEN=$(get_token "rhdh") if [[ 'component-*.yaml' == "${1}" ]]; then b_count=$(curl -s -k "$(backstage_url)/api/catalog/entity-facets?facet=kind" --cookie "$COOKIE" --cookie-jar "$COOKIE" -H 'Content-Type: application/json' -H 'Authorization: Bearer '"$ACCESS_TOKEN" | jq -r '.facets.kind[] | select(.value == "Component")| .count'); fi if [[ 'api-*.yaml' == "${1}" ]]; then b_count=$(curl -s -k "$(backstage_url)/api/catalog/entity-facets?facet=kind" --cookie "$COOKIE" --cookie-jar "$COOKIE" -H 'Content-Type: application/json' -H 'Authorization: Bearer '"$ACCESS_TOKEN" | jq -r '.facets.kind[] | select(.value == "API")| .count'); fi - if [[ $b_count -ge $e_count ]]; then break; fi + if [[ "$last_count" != "$b_count" ]] && [[ $last_count -ge 0 ]]; then # reset the timeout if current count changes + log_info "The current count changed, resetting entity waiting timeout to $timeout seconds" + timeout_timestamp=$(date -d "$timeout seconds" "+%s") + last_count=$b_count + fi + if [[ $b_count -ge $e_count ]]; then + log_info "The entity count reached expected value ($b_count)" + break + fi fi - echo "Waiting for the entity count to be ${e_count} (current: ${b_count})" + log_info "Waiting for the entity count to be ${e_count} (current: ${b_count})" sleep 10s done done @@ -159,26 +169,28 @@ create_group() { while ((attempt <= max_attempts)); do token=$(get_token) groupname="group${0}" - echo " g, group:default/${groupname}, role:default/perf_admin" >>"$TMP_DIR/group-rbac.yaml" - curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/groups" \ + response="$(curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/groups" \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer '"$token" \ - --data-raw '{"name": "'"${groupname}"'"}' |& tee -a "$TMP_DIR/create_group.log" - if [ "${PIPESTATUS[0]}" -eq 0 ]; then - echo "[INFO][$(date --utc -Ins)] Group $groupname created" >>"$TMP_DIR/create_group.log" + --data-raw '{"name": "'"${groupname}"'"}' 2>&1)" + if [ "${PIPESTATUS[0]}" -eq 0 ] && ! echo "$response" | grep -q 'error' >&/dev/null; then + log_info "Group $groupname created" >>"$TMP_DIR/create_group.log" return else - echo "[WARNING][$(date --utc -Ins)] Unable to create the $groupname group at $attempt. attempt. Trying again up to $max_attempts times." >>"$TMP_DIR/create_group.log" + log_warn "Unable to create the $groupname group at $attempt. attempt. [$response]. Trying again up to $max_attempts times." >>"$TMP_DIR/create_group.log" ((attempt++)) fi done if [[ $attempt -gt $max_attempts ]]; then - echo "[ERROR][$(date --utc -Ins)] Unable to create the $groupname group in $max_attempts attempts, giving up!" |& tee -a "$TMP_DIR/create_group.log" + log_error "Unable to create the $groupname group in $max_attempts attempts, giving up!" |& tee -a "$TMP_DIR/create_group.log" fi } create_groups() { - echo "Creating Groups in Keycloak" + log_info "Creating Groups in Keycloak" + for i in $(seq 1 "$GROUP_COUNT"); do + echo " g, group:default/group${i}, role:default/perf_admin" >>"$TMP_DIR/group-rbac.yaml" + done sleep 5 seq 1 "${GROUP_COUNT}" | xargs -n1 -P"${POPULATION_CONCURRENCY}" bash -c 'create_group' } @@ -192,45 +204,61 @@ create_user() { [[ $grp -eq 0 ]] && grp=${GROUP_COUNT} username="test${0}" groupname="group${grp}" - curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/users" \ + response="$(curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/users" \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer '"$token" \ - --data-raw '{"firstName":"'"${username}"'","lastName":"tester", "email":"'"${username}"'@test.com","emailVerified":"true", "enabled":"true", "username":"'"${username}"'","groups":["/'"${groupname}"'"],"credentials":[{"type":"password","value":"'"${KEYCLOAK_USER_PASS}"'","temporary":false}]}' |& tee -a "$TMP_DIR/create_user.log" - if [ "${PIPESTATUS[0]}" -eq 0 ]; then - echo "[INFO][$(date --utc -Ins)] User $username ($groupname) created" >>"$TMP_DIR/create_user.log" + --data-raw '{"firstName":"'"${username}"'","lastName":"tester", "email":"'"${username}"'@test.com","emailVerified":"true", "enabled":"true", "username":"'"${username}"'","groups":["/'"${groupname}"'"],"credentials":[{"type":"password","value":"'"${KEYCLOAK_USER_PASS}"'","temporary":false}]}' 2>&1)" + if [ "${PIPESTATUS[0]}" -eq 0 ] && ! echo "$response" | grep -q 'error' >&/dev/null; then + log_info "User $username ($groupname) created" >>"$TMP_DIR/create_user.log" return else - echo "[WARNING][$(date --utc -Ins)] Unable to create the $username user at $attempt. attempt. Trying again up to $max_attempts times." >>"$TMP_DIR/create_user.log" + log_warn "Unable to create the $username user at $attempt. attempt. [$response]. Trying again up to $max_attempts times." >>"$TMP_DIR/create_user.log" ((attempt++)) fi done if [[ $attempt -gt $max_attempts ]]; then - echo "[ERROR][$(date --utc -Ins)] Unable to create the $username user in $max_attempts attempts, giving up!" |& tee -a "$TMP_DIR/create_user.log" + log_error "Unable to create the $username user in $max_attempts attempts, giving up!" |& tee -a "$TMP_DIR/create_user.log" fi } create_users() { - echo "Creating Users in Keycloak" + log_info "Creating Users in Keycloak" export GROUP_COUNT sleep 5 seq 1 "${BACKSTAGE_USER_COUNT}" | xargs -n1 -P"${POPULATION_CONCURRENCY}" bash -c 'create_user' } token_lockfile="$TMP_DIR/token.lockfile" +log() { + echo "{\"level\":\"${2:-info}\",\"ts\":\"$(date --utc -Ins)\",\"message\":\"$1\"}" +} + +log_info() { + log "$1" "info" +} + +log_warn() { + log "$1" "warn" +} + +log_error() { + log "$1" "error" +} + log_token() { - echo "[${2:-INFO}][$(date --utc -Ins)] $1" >>"$TMP_DIR/get_token.log" + log "$1" "$2" >>"$TMP_DIR/get_token.log" } log_token_info() { - log_token "$1" "INFO" + log_token "$1" "info" } log_token_err() { - log_token "$1" "ERROR" + log_token "$1" "error" } keycloak_token() { - curl -s -k "$(keycloak_url)/auth/realms/master/protocol/openid-connect/token" -d username=admin -d "password=${keycloak_pass}" -d 'grant_type=password' -d 'client_id=admin-cli' | jq -r ".expires_in_timestamp = $(date -d '30 seconds' +%s)" + curl -s -k "$(keycloak_url)/auth/realms/master/protocol/openid-connect/token" -d username=admin -d "password=$1" -d 'grant_type=password' -d 'client_id=admin-cli' | jq -r ".expires_in_timestamp = $(date -d '30 seconds' +%s)" } rhdh_token() { @@ -280,7 +308,7 @@ rhdh_token() { --data-urlencode "code=$code" \ --data-urlencode "session_state=$session_state" \ --data-urlencode "state=$state" \ - "$CODE_URL" | jq -r ".backstageIdentity" | jq -r ".expires_in_timestamp = $(date -d '50 minutes' +%s)") + "$CODE_URL" | jq -r ".backstageIdentity" | jq -r ".expires_in_timestamp = $(date -d '30 minutes' +%s)") echo "$ACCESS_TOKEN" } @@ -288,8 +316,14 @@ get_token() { service=$1 if [[ ${service} == 'rhdh' ]]; then token_file="$TMP_DIR/rhdh_token.json" + token_field=".token" + token_type="RHDH" + token_timeout="3600" else token_file="$TMP_DIR/keycloak_token.json" + token_field=".access_token" + token_type="Keycloak" + token_timeout="60" fi while ! mkdir "$token_lockfile" 2>/dev/null; do sleep 0.5s @@ -297,35 +331,30 @@ get_token() { #shellcheck disable=SC2064 trap "rm -rf $token_lockfile; exit" INT TERM EXIT HUP - timeout_timestamp=$(date -d "60 seconds" "+%s") - while [ ! -f "$token_file" ] || [ ! -s "$token_file" ] || [ -z "$(jq -rc '.expires_in_timestamp' "$token_file")" ] || [ "$(date +%s)" -gt "$(jq -rc '.expires_in_timestamp' "$token_file")" ]; do - log_token_info "Refreshing keycloak token" + timeout_timestamp=$(date -d "$token_timeout seconds" "+%s") + while [ ! -f "$token_file" ] || [ ! -s "$token_file" ] || [ -z "$(jq -rc '.expires_in_timestamp' "$token_file")" ] || [ "$(date +%s)" -gt "$(jq -rc '.expires_in_timestamp' "$token_file")" ] || [ "$(jq -rc "$token_field" "$token_file")" == "null" ]; do if [ "$(date "+%s")" -gt "$timeout_timestamp" ]; then - log_token_err "Timeout getting keycloak token" + log_token_err "Timeout getting $token_type token" exit 1 fi + log_token_info "Refreshing $token_type token" if [[ ${service} == 'rhdh' ]]; then - log_token_info "Refreshing RHDH token" [[ -f "$token_file" ]] && rm -rf "$token_file" && rm -rf "$TMP_DIR/cookie.jar" - if ! rhdh_token >"$token_file"; then - log_token_err "Unable to get token, re-attempting" + if ! rhdh_token >"$token_file" || [ "$(jq -rc "$token_field" "$token_file")" == "null" ]; then + log_token_err "Unable to get $token_type token, re-attempting" fi else keycloak_pass=$(oc -n "${RHDH_NAMESPACE}" get secret credential-rhdh-sso -o template --template='{{.data.ADMIN_PASSWORD}}' | base64 -d) - if ! keycloak_token >"$token_file"; then - log_token_err "Unable to get token, re-attempting" + if ! keycloak_token "$keycloak_pass" >"$token_file"; then + log_token_err "Unable to get $token_type token, re-attempting" fi fi sleep 5s done - if [[ ${service} == 'rhdh' ]]; then - jq -rc '.token' "$token_file" - else - jq -rc '.access_token' "$token_file" - fi + jq -rc "$token_field" "$token_file" rm -rf "$token_lockfile" } -export -f keycloak_url backstage_url get_token keycloak_token rhdh_token create_group create_user log_token log_token_info log_token_err +export -f keycloak_url backstage_url get_token keycloak_token rhdh_token create_group create_user log log_info log_warn log_error log_token log_token_info log_token_err export kc_lockfile bs_lockfile token_lockfile diff --git a/ci-scripts/rhdh-setup/deploy.sh b/ci-scripts/rhdh-setup/deploy.sh index f58c275..366661f 100755 --- a/ci-scripts/rhdh-setup/deploy.sh +++ b/ci-scripts/rhdh-setup/deploy.sh @@ -80,10 +80,10 @@ wait_to_start_in_namespace() { interval=10s while ! /bin/bash -c "$cli -n $namespace get $rn -o name"; do if [ "$(date "+%s")" -gt "$timeout_timestamp" ]; then - echo "[ERROR][$(date --utc -Ins)] Timeout waiting for $description to start" + log_error "Timeout waiting for $description to start" exit 1 else - echo "[INFO][$(date --utc -Ins)] Waiting $interval for $description to start..." + log_info "Waiting $interval for $description to start..." sleep "$interval" fi done @@ -99,10 +99,10 @@ wait_for_crd() { interval=10s while ! /bin/bash -c "$cli get $rn"; do if [ "$(date "+%s")" -gt "$timeout_timestamp" ]; then - echo "[ERROR][$(date --utc -Ins)] Timeout waiting for $description to exist" + log_error "Timeout waiting for $description to exist" exit 1 else - echo "[INFO][$(date --utc -Ins)] Waiting $interval for $description to exist..." + log_info "Waiting $interval for $description to exist..." sleep "$interval" fi done @@ -150,13 +150,16 @@ install() { keycloak_install if $PRE_LOAD_DB; then - create_groups - create_users + log_info "Creating users and groups in Keycloak in background" + create_users_groups |& tee -a "${TMP_DIR}/create-users-groups.log" & fi - backstage_install + backstage_install |& tee -a "${TMP_DIR}/backstage-install.log" psql_debug setup_monitoring + + log_info "Waiting for all the users and groups to be created in Keycloak" + wait } keycloak_install() { @@ -187,23 +190,31 @@ keycloak_install() { envsubst