Skip to content

Commit

Permalink
Refine shell scripts to make shellcheck happy
Browse files Browse the repository at this point in the history
Signed-off-by: Pavel Macík <[email protected]>
  • Loading branch information
pmacik committed Nov 28, 2023
1 parent e4f48f5 commit d71646f
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 96 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ shellcheck:
## Run all linters
.PHONY: lint
lint: shellcheck
shellcheck $$(find -name '*.sh')

## === CI ===

Expand Down
12 changes: 7 additions & 5 deletions ci-scripts/collect-results.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ monitoring_collection_dir=$ARTIFACT_DIR/monitoring-collection-raw-data-dir
mkdir -p "$monitoring_collection_dir"

try_gather_file() {
if [ -f $1 ]; then
cp -vf $1 ${2:-$ARTIFACT_DIR}
if [ -f "$1" ]; then
cp -vf "$1" "${2:-$ARTIFACT_DIR}"
else
echo "WARNING: Tried to gather $1 but the file was not found!"
fi
}

try_gather_dir() {
if [ -d $1 ]; then
cp -rvf $1 ${2:-$ARTIFACT_DIR}
if [ -d "$1" ]; then
cp -rvf "$1" "${2:-$ARTIFACT_DIR}"
else
echo "WARNING: Tried to gather $1 but the directory was not found!"
fi
Expand All @@ -36,6 +36,7 @@ PYTHON_VENV_DIR=.venv
echo "$(date --utc -Ins) Setting up tool to collect monitoring data"
python3 -m venv $PYTHON_VENV_DIR
set +u
# shellcheck disable=SC1091
source $PYTHON_VENV_DIR/bin/activate
set -u
python3 -m pip install --quiet -U pip
Expand All @@ -46,12 +47,13 @@ set -u

echo "$(date --utc -Ins) Collecting monitoring data"
set +u
# shellcheck disable=SC1091
source $PYTHON_VENV_DIR/bin/activate
set -u
mstart=$(date --utc --date "$(cat benchmark-before)" --iso-8601=seconds)
mend=$(date --utc --date "$(cat benchmark-after)" --iso-8601=seconds)
mhost=$(kubectl -n openshift-monitoring get route -l app.kubernetes.io/name=thanos-query -o json | jq --raw-output '.items[0].spec.host')
mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' scenarios/$(cat benchmark-scenario).py)
mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' "scenarios/$(cat benchmark-scenario).py")
status_data.py \
--status-data-file "$monitoring_collection_data" \
--set \
Expand Down
105 changes: 55 additions & 50 deletions ci-scripts/rhdh-setup/create_resource.sh
Original file line number Diff line number Diff line change
@@ -1,127 +1,132 @@
#!/bin/bash

export TMP_DIR=$(readlink -m .tmp)
mkdir -p $TMP_DIR
export WORKDIR=$(readlink -m .)
export TMP_DIR WORKDIR

function keycloak_url() {
f=$TMP_DIR/keycloak.url
if [ ! -f $f ]; then
echo -n "https://$(oc get routes keycloak -n ${RHDH_NAMESPACE} -o jsonpath='{.spec.host}')" >$f
TMP_DIR=$(readlink -m .tmp)
mkdir -p "$TMP_DIR"
WORKDIR=$(readlink -m .)

keycloak_url() {
f="$TMP_DIR/keycloak.url"
if [ ! -f "$f" ]; then
echo -n "https://$(oc get routes keycloak -n "${RHDH_NAMESPACE}" -o jsonpath='{.spec.host}')" >"$f"
fi
cat $f
cat "$f"
}

function backstage_url() {
f=$TMP_DIR/backstage.url
if [ ! -f $f ]; then
echo -n "https://$(oc get routes ${RHDH_HELM_RELEASE_NAME}-developer-hub -n ${RHDH_NAMESPACE} -o jsonpath='{.spec.host}')" >$f
backstage_url() {
f="$TMP_DIR/backstage.url"
if [ ! -f "$f" ]; then
echo -n "https://$(oc get routes "${RHDH_HELM_RELEASE_NAME}-developer-hub" -n "${RHDH_NAMESPACE}" -o jsonpath='{.spec.host}')" >"$f"
fi
cat $f
cat "$f"
}

export keycloak_url backstage_url

function create_per_grp() {
create_per_grp() {
varname=$2
obj_count=${!varname}
if [[ -z ${!varname} ]]; then
echo "$varname is not set: Skipping $1 "
exit 1
fi
local iter_count=$(echo "(${obj_count}/${GROUP_COUNT})" | bc)
local mod=$(echo "(${obj_count}%${GROUP_COUNT})" | bc)
local iter_count mod
iter_count=$(echo "(${obj_count}/${GROUP_COUNT})" | bc)
mod=$(echo "(${obj_count}%${GROUP_COUNT})" | bc)

if [[ ! ${mod} -eq 0 ]]; then
iter_count=$(echo "${iter_count}+1" | bc)
fi
indx=0
for i in $(seq 1 $((${iter_count}))); do
for j in $(seq 1 $((${GROUP_COUNT}))); do
for _ in $(seq 1 "${iter_count}"); do
for g in $(seq 1 "${GROUP_COUNT}"); do
indx=$((1 + indx))
[[ ${obj_count} -lt $indx ]] && break
local out=$(${1} ${j} ${indx})
$1 "$g" "$indx"
done
done
}

function clone_and_upload() {
clone_and_upload() {
git_str="${GITHUB_USER}:${GITHUB_TOKEN}@github.com"
base_name=$(basename $GITHUB_REPO)
base_name=$(basename "$GITHUB_REPO")
git_dir=$TMP_DIR/${base_name}
git_repo=$(echo $GITHUB_REPO | sed -e "s/github.com/${git_str}/g")
[[ -d ${git_dir} ]] && rm -rf ${git_dir}
git clone $git_repo $git_dir
cd $git_dir
git_repo=${GITHUB_REPO//github.com/${git_str}}
[[ -d "${git_dir}" ]] && rm -rf "${git_dir}"
git clone "$git_repo" "$git_dir"
cd "$git_dir" || return
git config user.name "rhdh-performance-bot"
git config user.email [email protected]
tmp_branch=$(mktemp -u XXXXXXXXXX)
git checkout -b $tmp_branch
mv -vf ${1} .
filename=$(basename ${1})
git add $filename
git checkout -b "$tmp_branch"
mv -vf "$1" .
filename=$(basename "$1")
git add "$filename"
git commit -a -m "commit objects"
git push -f --set-upstream origin $tmp_branch
git push -f --set-upstream origin "$tmp_branch"
cd ..
sleep 5
upload_url=${GITHUB_REPO%.*}/blob/${tmp_branch}/${filename}
upload_url="${GITHUB_REPO%.*}/blob/${tmp_branch}/${filename}"
curl -k "$(backstage_url)/api/catalog/locations" -X POST -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' --data-raw '{"type":"url","target":"'"${upload_url}"'"}'
}

function create_api() {
# shellcheck disable=SC2016
create_api() {
export grp_indx=$1
export api_indx=$2
cat $WORKDIR/template/component/api.template | envsubst '${grp_indx} ${api_indx}' >>$TMP_DIR/api.yaml
envsubst '${grp_indx} ${api_indx}' <"$WORKDIR/template/component/api.template" >>"$TMP_DIR/api.yaml"
}

function create_cmp() {
# shellcheck disable=SC2016
create_cmp() {
export grp_indx=$1
export cmp_indx=$2
cat $WORKDIR/template/component/component.template | envsubst '${grp_indx} ${cmp_indx}' >>$TMP_DIR/component.yaml
envsubst '${grp_indx} ${cmp_indx}' <"$WORKDIR/template/component/component.template" >>"$TMP_DIR/component.yaml"
}

function create_group() {
create_group() {
token=$(get_token)
curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/groups" \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer '$token \
-H 'Authorization: Bearer '"$token" \
--data-raw '{"name": "group'"${0}"'"}'
}

function create_groups() {
create_groups() {
echo "Creating Groups in Keycloak"
refresh_pid=$!
sleep 5
export -f create_group
seq 1 ${GROUP_COUNT} | xargs -n1 -P10 bash -c 'create_group'
seq 1 "${GROUP_COUNT}" | xargs -n1 -P10 bash -c 'create_group'
kill $refresh_pid
}

function create_user() {
create_user() {
token=$(get_token)
grp=$(echo "${0}%${GROUP_COUNT}" | bc)
[[ $grp -eq 0 ]] && grp=${GROUP_COUNT}
curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/users" \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer '$token \
-H 'Authorization: Bearer '"$token" \
--data-raw '{"firstName":"test'"${0}"'","lastName":"tester", "email":"test'"${0}"'@test.com", "enabled":"true", "username":"test'"${0}"'","groups":["/group'"${grp}"'"]}'
}

function create_users() {
create_users() {
echo "Creating Users in Keycloak"
export GROUP_COUNT
refresh_pid=$!
sleep 5
export -f create_user
seq 1 ${BACKSTAGE_USER_COUNT} | xargs -n1 -P10 bash -c 'create_user'
seq 1 "${BACKSTAGE_USER_COUNT}" | xargs -n1 -P10 bash -c 'create_user'
kill $refresh_pid
}

function get_token() {
get_token() {
token_file=$TMP_DIR/token.json
if [ ! -f $token_file ] || [ $(date +%s) -gt $(jq -rc '.expires_in_timestamp' $token_file) ]; then
keycloak_pass=$(oc -n ${RHDH_NAMESPACE} get secret credential-example-sso -o template --template='{{.data.ADMIN_PASSWORD}}' | base64 -d)
curl -s -k $(keycloak_url)/auth/realms/master/protocol/openid-connect/token -d username=admin -d password=${keycloak_pass} -d 'grant_type=password' -d 'client_id=admin-cli' | jq -r ".expires_in_timestamp = $(date -d '30 seconds' +%s)" >$token_file
if [ ! -f "$token_file" ] || [ "$(date +%s)" -gt "$(jq -rc '.expires_in_timestamp' "$token_file")" ]; then
keycloak_pass=$(oc -n "${RHDH_NAMESPACE}" get secret credential-example-sso -o template --template='{{.data.ADMIN_PASSWORD}}' | base64 -d)
curl -s -k "$(keycloak_url)/auth/realms/master/protocol/openid-connect/token" -d username=admin -d "password=${keycloak_pass}" -d 'grant_type=password' -d 'client_id=admin-cli' | jq -r ".expires_in_timestamp = $(date -d '30 seconds' +%s)" >"$token_file"
fi
jq -rc '.access_token' $token_file
jq -rc '.access_token' "$token_file"
}

export -f keycloak_url backstage_url get_token
78 changes: 41 additions & 37 deletions ci-scripts/rhdh-setup/deploy.sh
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
#!/bin/bash
set -uo pipefail

# shellcheck disable=SC1091
source ./create_resource.sh

[ -z ${QUAY_TOKEN} ]
[ -z ${GITHUB_TOKEN} ]
[ -z ${GITHUB_USER} ]
[ -z ${GITHUB_REPO} ]
[ -z "${QUAY_TOKEN}" ]
[ -z "${GITHUB_TOKEN}" ]
[ -z "${GITHUB_USER}" ]
[ -z "${GITHUB_REPO}" ]

export RHDH_NAMESPACE=${RHDH_NAMESPACE:-rhdh-performance}
export RHDH_HELM_RELEASE_NAME=${RHDH_HELM_RELEASE_NAME:-rhdh}
Expand All @@ -32,28 +33,29 @@ export API_COUNT="${API_COUNT:-1}"
export COMPONENT_COUNT="${COMPONENT_COUNT:-1}"

delete() {
if ! $cli get ns $RHDH_NAMESPACE >/dev/null; then
if ! $cli get ns "$RHDH_NAMESPACE" >/dev/null; then
echo "$RHDH_NAMESPACE namespace does not exit... Skipping. "
return
fi
for cr in keycloakusers keycloakclients keycloakrealms keycloaks; do
for res in $($clin get $cr.keycloak.org -o name); do
$clin patch $res -p '{"metadata":{"finalizers":[]}}' --type=merge
$clin delete $res --wait
for res in $($clin get "$cr.keycloak.org" -o name); do
$clin patch "$res" -p '{"metadata":{"finalizers":[]}}' --type=merge
$clin delete "$res" --wait
done
done
helm uninstall ${RHDH_HELM_RELEASE_NAME} --namespace ${RHDH_NAMESPACE}
$clin delete pvc data-${RHDH_HELM_RELEASE_NAME}-postgresql-0 --ignore-not-found
$cli delete ns ${RHDH_NAMESPACE} --wait
helm uninstall "${RHDH_HELM_RELEASE_NAME}" --namespace "${RHDH_NAMESPACE}"
$clin delete pvc "data-${RHDH_HELM_RELEASE_NAME}-postgresql-0" --ignore-not-found
$cli delete ns "${RHDH_NAMESPACE}" --wait
}

keycloak_install() {
$cli create namespace ${RHDH_NAMESPACE} --dry-run=client -o yaml | $cli apply -f -
export KEYCLOAK_CLIENT_SECRET=$(mktemp -u XXXXXXXXXX)
cat template/keycloak/keycloak-op.yaml | envsubst | $clin apply -f -
$cli create namespace "${RHDH_NAMESPACE}" --dry-run=client -o yaml | $cli apply -f -
export KEYCLOAK_CLIENT_SECRET
KEYCLOAK_CLIENT_SECRET=$(mktemp -u XXXXXXXXXX)
envsubst <template/keycloak/keycloak-op.yaml | $clin apply -f -
grep -m 1 "rhsso-operator" <($clin get pods -w)
$clin wait --for=condition=Ready pod -l=name=rhsso-operator --timeout=300s
cat template/keycloak/keycloak.yaml | envsubst | $clin apply -f -
envsubst <template/keycloak/keycloak.yaml | $clin apply -f -
timeout=$(date -d "450 seconds" "+%s")
while ! /bin/bash -c "$clin get statefulset/keycloak -o name"; do
if [ "$(date "+%s")" -gt "$timeout" ]; then
Expand All @@ -65,29 +67,29 @@ keycloak_install() {
fi
done
$clin rollout status statefulset/keycloak --timeout=600s
cat template/keycloak/keycloakRealm.yaml | envsubst | $clin apply -f -
cat template/keycloak/keycloakClient.yaml | envsubst | $clin apply -f -
cat template/keycloak/keycloakUser.yaml | envsubst | $clin apply -f -
envsubst <template/keycloak/keycloakRealm.yaml | $clin apply -f -
envsubst <template/keycloak/keycloakClient.yaml | $clin apply -f -
envsubst <template/keycloak/keycloakUser.yaml | $clin apply -f -
}

# shellcheck disable=SC2016
backstage_install() {
until cat template/backstage/secret-rhdh-pull-secret.yaml | envsubst | $clin apply -f -; do $clin delete secret rhdh-pull-secret; done
cat template/backstage/app-config.yaml | envsubst '${RHDH_NAMESPACE} ${OPENSHIFT_APP_DOMAIN}' >app-config.yaml
until envsubst <template/backstage/secret-rhdh-pull-secret.yaml | $clin apply -f -; do $clin delete secret rhdh-pull-secret; done
envsubst '${RHDH_NAMESPACE} ${OPENSHIFT_APP_DOMAIN}' <template/backstage/app-config.yaml >app-config.yaml
until $clin create configmap app-config-rhdh --from-file "app-config-rhdh.yaml=app-config.yaml"; do $clin delete configmap app-config-rhdh; done
cat template/backstage/plugin-secrets.yaml | envsubst | $clin apply -f -
helm repo add rhdh-helm-repo ${RHDH_HELM_REPO}
envsubst <template/backstage/plugin-secrets.yaml | $clin apply -f -
helm repo add rhdh-helm-repo "${RHDH_HELM_REPO}"
helm repo update rhdh-helm-repo
cat template/backstage/chart-values.yaml |
envsubst \
'${OPENSHIFT_APP_DOMAIN} \
${RHDH_HELM_RELEASE_NAME} \
${RHDH_DEPLOYMENT_REPLICAS} \
${RHDH_DB_REPLICAS} \
${RHDH_IMAGE_REGISTRY} \
${RHDH_IMAGE_REPO} \
${RHDH_IMAGE_TAG} \
' | helm upgrade --install ${RHDH_HELM_RELEASE_NAME} --devel rhdh-helm-repo/${RHDH_HELM_CHART} -n ${RHDH_NAMESPACE} --values -
$clin rollout status deployment/${RHDH_HELM_RELEASE_NAME}-developer-hub --timeout=300s
envsubst \
'${OPENSHIFT_APP_DOMAIN} \
${RHDH_HELM_RELEASE_NAME} \
${RHDH_DEPLOYMENT_REPLICAS} \
${RHDH_DB_REPLICAS} \
${RHDH_IMAGE_REGISTRY} \
${RHDH_IMAGE_REPO} \
${RHDH_IMAGE_TAG} \
' <template/backstage/chart-values.yaml | helm upgrade --install "${RHDH_HELM_RELEASE_NAME}" --devel "rhdh-helm-repo/${RHDH_HELM_CHART}" -n "${RHDH_NAMESPACE}" --values -
$clin rollout status "deployment/${RHDH_HELM_RELEASE_NAME}-developer-hub" --timeout=300s
}

setup_monitoring() {
Expand Down Expand Up @@ -159,11 +161,13 @@ create_objs() {
fi

if [[ ${GITHUB_USER} ]] && [[ ${GITHUB_REPO} ]]; then
create_per_grp create_cmp COMPONENT_COUNT
[[ $? -eq 0 ]] && clone_and_upload $TMP_DIR/component.yaml
if create_per_grp create_cmp COMPONENT_COUNT; then
clone_and_upload "$TMP_DIR/component.yaml"
fi

create_per_grp create_api API_COUNT
[[ $? -eq 0 ]] && clone_and_upload $TMP_DIR/api.yaml
if create_per_grp create_api API_COUNT; then
clone_and_upload "$TMP_DIR/api.yaml"
fi
else
echo "skipping component creating. GITHUB_REPO and GITHUB_USER not set"
exit 1
Expand Down
8 changes: 4 additions & 4 deletions ci-scripts/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ echo -e "\n === Executing RHDH load test ===\n"
export SCENARIO

# testing env
export USERS=1000
export USERS HOST
USERS=1000
#export WORKERS=100
#export DURATION=10m
export SCENARIO="search-catalog"

export HOST="https://$(oc get routes rhdh-developer-hub -n rhdh-performance -o jsonpath='{.spec.host}')"
SCENARIO="search-catalog"
HOST="https://$(oc get routes rhdh-developer-hub -n rhdh-performance -o jsonpath='{.spec.host}')"
# end-of testing env

echo "$(date --utc -Ins) Running the test"
Expand Down

0 comments on commit d71646f

Please sign in to comment.