From 735ffac3465229f4fae6d39ec116da112e8d6a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Mac=C3=ADk?= Date: Wed, 10 Jan 2024 16:12:10 +0100 Subject: [PATCH] feat(RHIDP-968): Split component YAMLs to shards of 50k entities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Pavel Macík --- ci-scripts/collect-results.sh | 3 + ci-scripts/rhdh-setup/create_resource.sh | 73 ++++++++++++++++------- ci-scripts/rhdh-setup/deploy.sh | 4 +- ci-scripts/scalability/collect-results.sh | 31 ++++++---- 4 files changed, 73 insertions(+), 38 deletions(-) diff --git a/ci-scripts/collect-results.sh b/ci-scripts/collect-results.sh index 026f693..d155043 100755 --- a/ci-scripts/collect-results.sh +++ b/ci-scripts/collect-results.sh @@ -62,6 +62,9 @@ try_gather_file "${TMP_DIR}/populate-after" try_gather_file "${TMP_DIR}/benchmark-before" try_gather_file "${TMP_DIR}/benchmark-after" try_gather_file "${TMP_DIR}/benchmark-scenario" +try_gather_file "${TMP_DIR}/create_group.log" +try_gather_file "${TMP_DIR}/create_user.log" +try_gather_file "${TMP_DIR}/get_token.log" try_gather_file load-test.log PYTHON_VENV_DIR=.venv diff --git a/ci-scripts/rhdh-setup/create_resource.sh b/ci-scripts/rhdh-setup/create_resource.sh index cea6954..c1478db 100755 --- a/ci-scripts/rhdh-setup/create_resource.sh +++ b/ci-scripts/rhdh-setup/create_resource.sh @@ -3,6 +3,7 @@ export TMP_DIR WORKDIR POPULATION_CONCURRENCY=${POPULATION_CONCURRENCY:-10} +COMPONENT_SHARD_SIZE=${COMPONENT_SHARD_SIZE:-50000} TMP_DIR=${TMP_DIR:-$(readlink -m .tmp)} mkdir -p "$TMP_DIR" @@ -43,6 +44,7 @@ backstage_url() { } create_per_grp() { + echo "Creating entity YAML files" varname=$2 obj_count=${!varname} if [[ -z ${!varname} ]]; then @@ -57,16 +59,21 @@ create_per_grp() { iter_count=$(echo "${iter_count}+1" | bc) fi indx=0 + shard_index=0 for _ in $(seq 1 "${iter_count}"); do for g in $(seq 1 "${GROUP_COUNT}"); do indx=$((1 + indx)) [[ ${obj_count} -lt $indx ]] && break - $1 "$g" "$indx" + $1 "$g" "$indx" "$shard_index" + if [ "$(echo "(${indx}%${COMPONENT_SHARD_SIZE})" | bc)" == "0" ]; then + shard_index=$((shard_index + 1)) + fi done done } clone_and_upload() { + echo "Uploading entities to GitHub" git_str="${GITHUB_USER}:${GITHUB_TOKEN}@github.com" base_name=$(basename "$GITHUB_REPO") git_dir=$TMP_DIR/${base_name} @@ -78,29 +85,35 @@ clone_and_upload() { git config user.email rhdh-performance-bot@redhat.com tmp_branch=$(mktemp -u XXXXXXXXXX) git checkout -b "$tmp_branch" - mv -vf "$1" . - filename=$(basename "$1") - git add "$filename" + mapfile -t files < <(find "$TMP_DIR" -name "$1") + for filename in "${files[@]}"; do + mv -vf "$filename" "$(basename "$filename")" + git add "$(basename "$filename")" + done git commit -a -m "commit objects" git push -f --set-upstream origin "$tmp_branch" cd .. sleep 5 - upload_url="${GITHUB_REPO%.*}/blob/${tmp_branch}/${filename}" - curl -k "$(backstage_url)/api/catalog/locations" -X POST -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' --data-raw '{"type":"url","target":"'"${upload_url}"'"}' + for filename in "${files[@]}"; do + upload_url="${GITHUB_REPO%.*}/blob/${tmp_branch}/$(basename "$filename")" + curl -k "$(backstage_url)/api/catalog/locations" -X POST -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' --data-raw '{"type":"url","target":"'"${upload_url}"'"}' + done } # shellcheck disable=SC2016 create_api() { export grp_indx=$1 export api_indx=$2 - envsubst '${grp_indx} ${api_indx}' <"$WORKDIR/template/component/api.template" >>"$TMP_DIR/api.yaml" + export shard_indx=${3:-0} + envsubst '${grp_indx} ${api_indx}' <"$WORKDIR/template/component/api.template" >>"$TMP_DIR/api-$shard_indx.yaml" } # shellcheck disable=SC2016 create_cmp() { export grp_indx=$1 export cmp_indx=$2 - envsubst '${grp_indx} ${cmp_indx}' <"$WORKDIR/template/component/component.template" >>"$TMP_DIR/component.yaml" + export shard_indx=${3:-0} + envsubst '${grp_indx} ${cmp_indx}' <"$WORKDIR/template/component/component.template" >>"$TMP_DIR/component-$shard_indx.yaml" } create_group() { @@ -109,15 +122,14 @@ create_group() { curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/groups" \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer '"$token" \ - --data-raw '{"name": "'"${groupname}"'"}' + --data-raw '{"name": "'"${groupname}"'"}' |& tee -a "$TMP_DIR/create_group.log" + echo "Group $groupname created" >>"$TMP_DIR/create_group.log" } create_groups() { echo "Creating Groups in Keycloak" - refresh_pid=$! sleep 5 seq 1 "${GROUP_COUNT}" | xargs -n1 -P"${POPULATION_CONCURRENCY}" bash -c 'create_group' - kill $refresh_pid } create_user() { @@ -129,34 +141,49 @@ create_user() { curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/users" \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer '"$token" \ - --data-raw '{"firstName":"'"${username}"'","lastName":"tester", "email":"'"${username}"'@test.com", "enabled":"true", "username":"'"${username}"'","groups":["/'"${groupname}"'"]}' + --data-raw '{"firstName":"'"${username}"'","lastName":"tester", "email":"'"${username}"'@test.com", "enabled":"true", "username":"'"${username}"'","groups":["/'"${groupname}"'"]}' |& tee -a "$TMP_DIR/create_user.log" + echo "User $username ($groupname) created" >>"$TMP_DIR/create_user.log" } create_users() { echo "Creating Users in Keycloak" export GROUP_COUNT - refresh_pid=$! sleep 5 seq 1 "${BACKSTAGE_USER_COUNT}" | xargs -n1 -P"${POPULATION_CONCURRENCY}" bash -c 'create_user' - kill $refresh_pid } token_lockfile="$TMP_DIR/token.lockfile" +log_token() { + token_log="$TMP_DIR/get_token.log" + echo "[$(date --utc -Ins)] $1" >>"$token_log" +} get_token() { + token_log="$TMP_DIR/get_token.log" token_file=$TMP_DIR/token.json - exec 3>"$token_lockfile" - flock 3 || { - echo "Failed to acquire lock" - exit 1 - } - if [ ! -f "$token_file" ] || [ ! -s "$token_file" ] || [ "$(date +%s)" -gt "$(jq -rc '.expires_in_timestamp' "$token_file")" ]; then + while ! mkdir "$token_lockfile" 2>/dev/null; do + sleep 0.5s + done + #shellcheck disable=SC2064 + trap "rm -rf $token_lockfile; exit" INT TERM EXIT HUP + + timeout_timestamp=$(date -d "60 seconds" "+%s") + while [ ! -f "$token_file" ] || [ ! -s "$token_file" ] || [ "$(date +%s)" -gt "$(jq -rc '.expires_in_timestamp' "$token_file")" ]; do + log_token "refreshing keycloak token" keycloak_pass=$(oc -n "${RHDH_NAMESPACE}" get secret credential-example-sso -o template --template='{{.data.ADMIN_PASSWORD}}' | base64 -d) curl -s -k "$(keycloak_url)/auth/realms/master/protocol/openid-connect/token" -d username=admin -d "password=${keycloak_pass}" -d 'grant_type=password' -d 'client_id=admin-cli' | jq -r ".expires_in_timestamp = $(date -d '30 seconds' +%s)" >"$token_file" - fi - flock -u 3 + if [ "$(date "+%s")" -gt "$timeout_timestamp" ]; then + log_token "ERROR: Timeout getting keycloak token" + exit 1 + else + log_token "Re-attempting to get keycloak token" + sleep 5s + fi + done + + rm -rf "$token_lockfile" jq -rc '.access_token' "$token_file" } -export -f keycloak_url backstage_url backstage_url get_token create_group create_user +export -f keycloak_url backstage_url backstage_url get_token create_group create_user log_token export kc_lockfile bs_lockfile token_lockfile diff --git a/ci-scripts/rhdh-setup/deploy.sh b/ci-scripts/rhdh-setup/deploy.sh index b73fa59..d0aa82c 100755 --- a/ci-scripts/rhdh-setup/deploy.sh +++ b/ci-scripts/rhdh-setup/deploy.sh @@ -193,11 +193,11 @@ create_objs() { if [[ ${GITHUB_USER} ]] && [[ ${GITHUB_REPO} ]]; then if create_per_grp create_cmp COMPONENT_COUNT; then - clone_and_upload "$TMP_DIR/component.yaml" + clone_and_upload "component-*.yaml" fi if create_per_grp create_api API_COUNT; then - clone_and_upload "$TMP_DIR/api.yaml" + clone_and_upload "api-*.yaml" fi else echo "skipping component creating. GITHUB_REPO and GITHUB_USER not set" diff --git a/ci-scripts/scalability/collect-results.sh b/ci-scripts/scalability/collect-results.sh index 67dc938..d5aa52a 100755 --- a/ci-scripts/scalability/collect-results.sh +++ b/ci-scripts/scalability/collect-results.sh @@ -35,23 +35,28 @@ for w in "${workers[@]}"; do IFS=":" read -ra tokens <<<"${au_sr}" active_users=${tokens[0]} output="$ARTIFACT_DIR/scalability_c-${r}r-db_${s}-${bu}bu-${bg}bg-${w}w-${active_users}u.csv" - echo "CatalogSize${csv_delim}AverateRPS${csv_delim}MaxRPS${csv_delim}AverageRT${csv_delim}MaxRT${csv_delim}FailRate${csv_delim}DBStorageUsed${csv_delim}DBStorageAvailable${csv_delim}DBStorageCapacity" >"$output" + header="CatalogSize${csv_delim}AverateRPS${csv_delim}MaxRPS${csv_delim}AverageRT${csv_delim}MaxRT${csv_delim}FailRate${csv_delim}DBStorageUsed${csv_delim}DBStorageAvailable${csv_delim}DBStorageCapacity" + echo "$header" >"$output" for c in "${catalog_sizes[@]}"; do index="${r}r-db_${s}-${bu}bu-${bg}bg-${w}w-${c}c" - benchmark_json="$(find . -name benchmark.json | grep "$index" || true)" - echo -n "$c;" >>"$output" + benchmark_json="$(find "${ARTIFACT_DIR}" -name benchmark.json | grep "$index" || true)" + echo -n "$c" >>"$output" if [ -n "$benchmark_json" ]; then - jq_cmd="(.results.\"locust-operator\".locust_requests_current_rps_Aggregated.mean | tostring) \ - + $csv_delim_quoted + (.results.\"locust-operator\".locust_requests_current_rps_Aggregated.max | tostring) \ - + $csv_delim_quoted + (.results.\"locust-operator\".locust_requests_avg_response_time_Aggregated.mean | tostring) \ - + $csv_delim_quoted + (.results.\"locust-operator\".locust_requests_avg_response_time_Aggregated.max | tostring) \ - + $csv_delim_quoted + (.results.\"locust-operator\".locust_requests_fail_ratio_Aggregated.mean | tostring) \ - + $csv_delim_quoted + (.measurements.cluster.pv_stats.populate.\"data-rhdh-postgresql-primary-0\".used_bytes.max | tostring) \ - + $csv_delim_quoted + (.measurements.cluster.pv_stats.populate.\"data-rhdh-postgresql-primary-0\".available_bytes.min | tostring) \ - + $csv_delim_quoted + (.measurements.cluster.pv_stats.populate.\"data-rhdh-postgresql-primary-0\".capacity_bytes.max | tostring)" - sed -Ee 's/: ([0-9]+\.[0-9]*[X]+[0-9e\+-]*|[0-9]*X+[0-9]*\.[0-9e\+-]*|[0-9]*X+[0-9]*\.[0-9]*X+[0-9e\+-]+),/: "\1",/g' "$benchmark_json" | jq -rc "$jq_cmd" >>"$output" + echo "Gathering data from $benchmark_json" + jq_cmd="$csv_delim_quoted + (.results.\"locust-operator\".locust_requests_current_rps_Aggregated.mean | tostring) \ + + $csv_delim_quoted + (.results.\"locust-operator\".locust_requests_current_rps_Aggregated.max | tostring) \ + + $csv_delim_quoted + (.results.\"locust-operator\".locust_requests_avg_response_time_Aggregated.mean | tostring) \ + + $csv_delim_quoted + (.results.\"locust-operator\".locust_requests_avg_response_time_Aggregated.max | tostring) \ + + $csv_delim_quoted + (.results.\"locust-operator\".locust_requests_fail_ratio_Aggregated.mean | tostring) \ + + $csv_delim_quoted + (.measurements.cluster.pv_stats.populate.\"data-rhdh-postgresql-primary-0\".used_bytes.max | tostring) \ + + $csv_delim_quoted + (.measurements.cluster.pv_stats.populate.\"data-rhdh-postgresql-primary-0\".available_bytes.min | tostring) \ + + $csv_delim_quoted + (.measurements.cluster.pv_stats.populate.\"data-rhdh-postgresql-primary-0\".capacity_bytes.max | tostring)" + sed -Ee 's/: ([0-9]+\.[0-9]*[X]+[0-9e\+-]*|[0-9]*X+[0-9]*\.[0-9e\+-]*|[0-9]*X+[0-9]*\.[0-9]*X+[0-9e\+-]+)/: "\1"/g' "$benchmark_json" | jq -rc "$jq_cmd" >>"$output" else - echo ";" >>"$output" + for i in $(seq 1 $(echo "$header" | tr -cd "$csv_delim" | wc -c)); do + echo -n ";" >>"$output" + done + echo >>"$output" fi done done