From d71646f9b2df484e402b2b6f7dc9437d691f2e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Mac=C3=ADk?= Date: Tue, 28 Nov 2023 13:46:57 +0100 Subject: [PATCH] Refine shell scripts to make shellcheck happy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Pavel Macík --- Makefile | 1 + ci-scripts/collect-results.sh | 12 +-- ci-scripts/rhdh-setup/create_resource.sh | 105 ++++++++++++----------- ci-scripts/rhdh-setup/deploy.sh | 78 +++++++++-------- ci-scripts/test.sh | 8 +- 5 files changed, 108 insertions(+), 96 deletions(-) diff --git a/Makefile b/Makefile index b61f109..25360e8 100644 --- a/Makefile +++ b/Makefile @@ -137,6 +137,7 @@ shellcheck: ## Run all linters .PHONY: lint lint: shellcheck + shellcheck $$(find -name '*.sh') ## === CI === diff --git a/ci-scripts/collect-results.sh b/ci-scripts/collect-results.sh index 63a110d..df9d94e 100755 --- a/ci-scripts/collect-results.sh +++ b/ci-scripts/collect-results.sh @@ -13,16 +13,16 @@ monitoring_collection_dir=$ARTIFACT_DIR/monitoring-collection-raw-data-dir mkdir -p "$monitoring_collection_dir" try_gather_file() { - if [ -f $1 ]; then - cp -vf $1 ${2:-$ARTIFACT_DIR} + if [ -f "$1" ]; then + cp -vf "$1" "${2:-$ARTIFACT_DIR}" else echo "WARNING: Tried to gather $1 but the file was not found!" fi } try_gather_dir() { - if [ -d $1 ]; then - cp -rvf $1 ${2:-$ARTIFACT_DIR} + if [ -d "$1" ]; then + cp -rvf "$1" "${2:-$ARTIFACT_DIR}" else echo "WARNING: Tried to gather $1 but the directory was not found!" fi @@ -36,6 +36,7 @@ PYTHON_VENV_DIR=.venv echo "$(date --utc -Ins) Setting up tool to collect monitoring data" python3 -m venv $PYTHON_VENV_DIR set +u +# shellcheck disable=SC1091 source $PYTHON_VENV_DIR/bin/activate set -u python3 -m pip install --quiet -U pip @@ -46,12 +47,13 @@ set -u echo "$(date --utc -Ins) Collecting monitoring data" set +u +# shellcheck disable=SC1091 source $PYTHON_VENV_DIR/bin/activate set -u mstart=$(date --utc --date "$(cat benchmark-before)" --iso-8601=seconds) mend=$(date --utc --date "$(cat benchmark-after)" --iso-8601=seconds) mhost=$(kubectl -n openshift-monitoring get route -l app.kubernetes.io/name=thanos-query -o json | jq --raw-output '.items[0].spec.host') -mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' scenarios/$(cat benchmark-scenario).py) +mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' "scenarios/$(cat benchmark-scenario).py") status_data.py \ --status-data-file "$monitoring_collection_data" \ --set \ diff --git a/ci-scripts/rhdh-setup/create_resource.sh b/ci-scripts/rhdh-setup/create_resource.sh index 0cbe6fc..bf27248 100755 --- a/ci-scripts/rhdh-setup/create_resource.sh +++ b/ci-scripts/rhdh-setup/create_resource.sh @@ -1,127 +1,132 @@ #!/bin/bash -export TMP_DIR=$(readlink -m .tmp) -mkdir -p $TMP_DIR -export WORKDIR=$(readlink -m .) +export TMP_DIR WORKDIR -function keycloak_url() { - f=$TMP_DIR/keycloak.url - if [ ! -f $f ]; then - echo -n "https://$(oc get routes keycloak -n ${RHDH_NAMESPACE} -o jsonpath='{.spec.host}')" >$f +TMP_DIR=$(readlink -m .tmp) +mkdir -p "$TMP_DIR" +WORKDIR=$(readlink -m .) + +keycloak_url() { + f="$TMP_DIR/keycloak.url" + if [ ! -f "$f" ]; then + echo -n "https://$(oc get routes keycloak -n "${RHDH_NAMESPACE}" -o jsonpath='{.spec.host}')" >"$f" fi - cat $f + cat "$f" } -function backstage_url() { - f=$TMP_DIR/backstage.url - if [ ! -f $f ]; then - echo -n "https://$(oc get routes ${RHDH_HELM_RELEASE_NAME}-developer-hub -n ${RHDH_NAMESPACE} -o jsonpath='{.spec.host}')" >$f +backstage_url() { + f="$TMP_DIR/backstage.url" + if [ ! -f "$f" ]; then + echo -n "https://$(oc get routes "${RHDH_HELM_RELEASE_NAME}-developer-hub" -n "${RHDH_NAMESPACE}" -o jsonpath='{.spec.host}')" >"$f" fi - cat $f + cat "$f" } -export keycloak_url backstage_url - -function create_per_grp() { +create_per_grp() { varname=$2 obj_count=${!varname} if [[ -z ${!varname} ]]; then echo "$varname is not set: Skipping $1 " exit 1 fi - local iter_count=$(echo "(${obj_count}/${GROUP_COUNT})" | bc) - local mod=$(echo "(${obj_count}%${GROUP_COUNT})" | bc) + local iter_count mod + iter_count=$(echo "(${obj_count}/${GROUP_COUNT})" | bc) + mod=$(echo "(${obj_count}%${GROUP_COUNT})" | bc) if [[ ! ${mod} -eq 0 ]]; then iter_count=$(echo "${iter_count}+1" | bc) fi indx=0 - for i in $(seq 1 $((${iter_count}))); do - for j in $(seq 1 $((${GROUP_COUNT}))); do + for _ in $(seq 1 "${iter_count}"); do + for g in $(seq 1 "${GROUP_COUNT}"); do indx=$((1 + indx)) [[ ${obj_count} -lt $indx ]] && break - local out=$(${1} ${j} ${indx}) + $1 "$g" "$indx" done done } -function clone_and_upload() { +clone_and_upload() { git_str="${GITHUB_USER}:${GITHUB_TOKEN}@github.com" - base_name=$(basename $GITHUB_REPO) + base_name=$(basename "$GITHUB_REPO") git_dir=$TMP_DIR/${base_name} - git_repo=$(echo $GITHUB_REPO | sed -e "s/github.com/${git_str}/g") - [[ -d ${git_dir} ]] && rm -rf ${git_dir} - git clone $git_repo $git_dir - cd $git_dir + git_repo=${GITHUB_REPO//github.com/${git_str}} + [[ -d "${git_dir}" ]] && rm -rf "${git_dir}" + git clone "$git_repo" "$git_dir" + cd "$git_dir" || return git config user.name "rhdh-performance-bot" git config user.email rhdh-performance-bot@redhat.com tmp_branch=$(mktemp -u XXXXXXXXXX) - git checkout -b $tmp_branch - mv -vf ${1} . - filename=$(basename ${1}) - git add $filename + git checkout -b "$tmp_branch" + mv -vf "$1" . + filename=$(basename "$1") + git add "$filename" git commit -a -m "commit objects" - git push -f --set-upstream origin $tmp_branch + git push -f --set-upstream origin "$tmp_branch" cd .. sleep 5 - upload_url=${GITHUB_REPO%.*}/blob/${tmp_branch}/${filename} + upload_url="${GITHUB_REPO%.*}/blob/${tmp_branch}/${filename}" curl -k "$(backstage_url)/api/catalog/locations" -X POST -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' --data-raw '{"type":"url","target":"'"${upload_url}"'"}' } -function create_api() { +# shellcheck disable=SC2016 +create_api() { export grp_indx=$1 export api_indx=$2 - cat $WORKDIR/template/component/api.template | envsubst '${grp_indx} ${api_indx}' >>$TMP_DIR/api.yaml + envsubst '${grp_indx} ${api_indx}' <"$WORKDIR/template/component/api.template" >>"$TMP_DIR/api.yaml" } -function create_cmp() { +# shellcheck disable=SC2016 +create_cmp() { export grp_indx=$1 export cmp_indx=$2 - cat $WORKDIR/template/component/component.template | envsubst '${grp_indx} ${cmp_indx}' >>$TMP_DIR/component.yaml + envsubst '${grp_indx} ${cmp_indx}' <"$WORKDIR/template/component/component.template" >>"$TMP_DIR/component.yaml" } -function create_group() { +create_group() { token=$(get_token) curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/groups" \ -H 'Content-Type: application/json' \ - -H 'Authorization: Bearer '$token \ + -H 'Authorization: Bearer '"$token" \ --data-raw '{"name": "group'"${0}"'"}' } -function create_groups() { +create_groups() { echo "Creating Groups in Keycloak" refresh_pid=$! sleep 5 export -f create_group - seq 1 ${GROUP_COUNT} | xargs -n1 -P10 bash -c 'create_group' + seq 1 "${GROUP_COUNT}" | xargs -n1 -P10 bash -c 'create_group' kill $refresh_pid } -function create_user() { +create_user() { token=$(get_token) grp=$(echo "${0}%${GROUP_COUNT}" | bc) [[ $grp -eq 0 ]] && grp=${GROUP_COUNT} curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/users" \ -H 'Content-Type: application/json' \ - -H 'Authorization: Bearer '$token \ + -H 'Authorization: Bearer '"$token" \ --data-raw '{"firstName":"test'"${0}"'","lastName":"tester", "email":"test'"${0}"'@test.com", "enabled":"true", "username":"test'"${0}"'","groups":["/group'"${grp}"'"]}' } -function create_users() { +create_users() { echo "Creating Users in Keycloak" export GROUP_COUNT refresh_pid=$! sleep 5 export -f create_user - seq 1 ${BACKSTAGE_USER_COUNT} | xargs -n1 -P10 bash -c 'create_user' + seq 1 "${BACKSTAGE_USER_COUNT}" | xargs -n1 -P10 bash -c 'create_user' kill $refresh_pid } -function get_token() { +get_token() { token_file=$TMP_DIR/token.json - if [ ! -f $token_file ] || [ $(date +%s) -gt $(jq -rc '.expires_in_timestamp' $token_file) ]; then - keycloak_pass=$(oc -n ${RHDH_NAMESPACE} get secret credential-example-sso -o template --template='{{.data.ADMIN_PASSWORD}}' | base64 -d) - curl -s -k $(keycloak_url)/auth/realms/master/protocol/openid-connect/token -d username=admin -d password=${keycloak_pass} -d 'grant_type=password' -d 'client_id=admin-cli' | jq -r ".expires_in_timestamp = $(date -d '30 seconds' +%s)" >$token_file + if [ ! -f "$token_file" ] || [ "$(date +%s)" -gt "$(jq -rc '.expires_in_timestamp' "$token_file")" ]; then + keycloak_pass=$(oc -n "${RHDH_NAMESPACE}" get secret credential-example-sso -o template --template='{{.data.ADMIN_PASSWORD}}' | base64 -d) + curl -s -k "$(keycloak_url)/auth/realms/master/protocol/openid-connect/token" -d username=admin -d "password=${keycloak_pass}" -d 'grant_type=password' -d 'client_id=admin-cli' | jq -r ".expires_in_timestamp = $(date -d '30 seconds' +%s)" >"$token_file" fi - jq -rc '.access_token' $token_file + jq -rc '.access_token' "$token_file" } + +export -f keycloak_url backstage_url get_token diff --git a/ci-scripts/rhdh-setup/deploy.sh b/ci-scripts/rhdh-setup/deploy.sh index 6f86385..1c8658e 100755 --- a/ci-scripts/rhdh-setup/deploy.sh +++ b/ci-scripts/rhdh-setup/deploy.sh @@ -1,12 +1,13 @@ #!/bin/bash set -uo pipefail +# shellcheck disable=SC1091 source ./create_resource.sh -[ -z ${QUAY_TOKEN} ] -[ -z ${GITHUB_TOKEN} ] -[ -z ${GITHUB_USER} ] -[ -z ${GITHUB_REPO} ] +[ -z "${QUAY_TOKEN}" ] +[ -z "${GITHUB_TOKEN}" ] +[ -z "${GITHUB_USER}" ] +[ -z "${GITHUB_REPO}" ] export RHDH_NAMESPACE=${RHDH_NAMESPACE:-rhdh-performance} export RHDH_HELM_RELEASE_NAME=${RHDH_HELM_RELEASE_NAME:-rhdh} @@ -32,28 +33,29 @@ export API_COUNT="${API_COUNT:-1}" export COMPONENT_COUNT="${COMPONENT_COUNT:-1}" delete() { - if ! $cli get ns $RHDH_NAMESPACE >/dev/null; then + if ! $cli get ns "$RHDH_NAMESPACE" >/dev/null; then echo "$RHDH_NAMESPACE namespace does not exit... Skipping. " return fi for cr in keycloakusers keycloakclients keycloakrealms keycloaks; do - for res in $($clin get $cr.keycloak.org -o name); do - $clin patch $res -p '{"metadata":{"finalizers":[]}}' --type=merge - $clin delete $res --wait + for res in $($clin get "$cr.keycloak.org" -o name); do + $clin patch "$res" -p '{"metadata":{"finalizers":[]}}' --type=merge + $clin delete "$res" --wait done done - helm uninstall ${RHDH_HELM_RELEASE_NAME} --namespace ${RHDH_NAMESPACE} - $clin delete pvc data-${RHDH_HELM_RELEASE_NAME}-postgresql-0 --ignore-not-found - $cli delete ns ${RHDH_NAMESPACE} --wait + helm uninstall "${RHDH_HELM_RELEASE_NAME}" --namespace "${RHDH_NAMESPACE}" + $clin delete pvc "data-${RHDH_HELM_RELEASE_NAME}-postgresql-0" --ignore-not-found + $cli delete ns "${RHDH_NAMESPACE}" --wait } keycloak_install() { - $cli create namespace ${RHDH_NAMESPACE} --dry-run=client -o yaml | $cli apply -f - - export KEYCLOAK_CLIENT_SECRET=$(mktemp -u XXXXXXXXXX) - cat template/keycloak/keycloak-op.yaml | envsubst | $clin apply -f - + $cli create namespace "${RHDH_NAMESPACE}" --dry-run=client -o yaml | $cli apply -f - + export KEYCLOAK_CLIENT_SECRET + KEYCLOAK_CLIENT_SECRET=$(mktemp -u XXXXXXXXXX) + envsubst