From a7f77bfb024f371971c9a354309d2e88d56d21e6 Mon Sep 17 00:00:00 2001 From: yogananth subramanian Date: Tue, 4 Jun 2024 12:16:22 +0530 Subject: [PATCH 1/3] Enable postgresql logging This patch enables logging of long running queries in postgresql. --- Makefile | 5 +++++ ci-scripts/collect-results.sh | 1 + ci-scripts/rhdh-setup/deploy.sh | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/Makefile b/Makefile index 663a7d7..8625aad 100644 --- a/Makefile +++ b/Makefile @@ -192,6 +192,11 @@ endif @echo "Getting locust master log:" kubectl logs --namespace $(LOCUST_NAMESPACE) -f -l performance-test-pod-name=$(SCENARIO)-test-master | tee load-test.log date --utc -Ins>$(TMP_DIR)/benchmark-after +ifeq ($(RHDH_INSTALL_METHOD),helm) + oc exec rhdh-postgresql-primary-0 -n rhdh-performance -- sh -c 'cat /var/lib/pgsql/data/userdata/log/postgresql*.log'>postgresql.log +else + oc exec backstage-psql-developer-hub-0 -n rhdh-performance-olm -- sh -c 'cat /var/lib/pgsql/data/userdata/log/postgresql*.log'>postgresql.log +endif @echo "All done!!!" ## Run the scalability test diff --git a/ci-scripts/collect-results.sh b/ci-scripts/collect-results.sh index 9c8fa26..65070d4 100755 --- a/ci-scripts/collect-results.sh +++ b/ci-scripts/collect-results.sh @@ -70,6 +70,7 @@ try_gather_file "${TMP_DIR}/create_group.log" try_gather_file "${TMP_DIR}/create_user.log" try_gather_file "${TMP_DIR}/get_token.log" try_gather_file load-test.log +try_gather_file postgresql.log PYTHON_VENV_DIR=.venv diff --git a/ci-scripts/rhdh-setup/deploy.sh b/ci-scripts/rhdh-setup/deploy.sh index 8267e0e..2329bc4 100755 --- a/ci-scripts/rhdh-setup/deploy.sh +++ b/ci-scripts/rhdh-setup/deploy.sh @@ -55,6 +55,11 @@ export KEYCLOAK_USER_PASS=${KEYCLOAK_USER_PASS:-$(mktemp -u XXXXXXXXXX)} export AUTH_PROVIDER="${AUTH_PROVIDER:-''}" export ENABLE_RBAC="${ENABLE_RBAC:-false}" +export PSQL_LOG="${PSQL_LOG:-true}" +export LOG_MIN_DURATION_STATEMENT="${LOG_MIN_DURATION_STATEMENT:-65}" +export LOG_MIN_DURATION_SAMPLE="${LOG_MIN_DURATION_SAMPLE:-50}" +export LOG_STATEMENT_SAMPLE_RATE="${LOG_STATEMENT_SAMPLE_RATE:-0.7}" + export INSTALL_METHOD=helm TMP_DIR=$(readlink -m "${TMP_DIR:-.tmp}") @@ -147,6 +152,7 @@ install() { fi backstage_install + psql_debug setup_monitoring } @@ -284,6 +290,21 @@ install_rhdh_with_olm() { wait_to_start deployment "backstage-developer-hub" 300 300 } +# shellcheck disable=SC2016,SC1001,SC2086 +psql_debug() { + if [ "$INSTALL_METHOD" == "helm" ]; then + psql_db=rhdh-postgresql-primary-0 + elif [ "$INSTALL_METHOD" == "olm" ]; then + psql_db=backstage-psql-developer-hub-0 + fi + if ${PSQL_LOG}; then + oc exec "${psql_db}" -n "${RHDH_NAMESPACE}" -- sh -c "sed -i "s/^\s*#log_min_duration_statement.*/log_min_duration_statement=${LOG_MIN_DURATION_STATEMENT}/" /var/lib/pgsql/data/userdata/postgresql.conf " + oc exec "${psql_db}" -n "${RHDH_NAMESPACE}" -- sh -c "sed -i "s/^\s*#log_min_duration_sample.*/log_min_duration_sample=${LOG_MIN_DURATION_SAMPLE}/" /var/lib/pgsql/data/userdata/postgresql.conf " + oc exec "${psql_db}" -n "${RHDH_NAMESPACE}" -- sh -c "sed -i "s/^\s*#log_statement_sample_rate.*/log_statement_sample_rate=${LOG_STATEMENT_SAMPLE_RATE}/" /var/lib/pgsql/data/userdata/postgresql.conf " + fi + oc exec "${psql_db}" -n "${RHDH_NAMESPACE}" -- sh -c 'pg_ctl -D $PGDATA restart -mf' +} + setup_monitoring() { echo "Enabling user workload monitoring" rm -f config.yaml From 3eec03f0020eeef44ffb25661755141983e3f748 Mon Sep 17 00:00:00 2001 From: yogananth subramanian Date: Fri, 19 Jul 2024 14:57:15 +0530 Subject: [PATCH 2/3] Update Makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pavel Macík --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8625aad..25cbfd1 100644 --- a/Makefile +++ b/Makefile @@ -193,9 +193,9 @@ endif kubectl logs --namespace $(LOCUST_NAMESPACE) -f -l performance-test-pod-name=$(SCENARIO)-test-master | tee load-test.log date --utc -Ins>$(TMP_DIR)/benchmark-after ifeq ($(RHDH_INSTALL_METHOD),helm) - oc exec rhdh-postgresql-primary-0 -n rhdh-performance -- sh -c 'cat /var/lib/pgsql/data/userdata/log/postgresql*.log'>postgresql.log + oc exec rhdh-postgresql-primary-0 -n $(RHDH_NAMESPACE) -- sh -c 'cat /var/lib/pgsql/data/userdata/log/postgresql*.log'>postgresql.log else - oc exec backstage-psql-developer-hub-0 -n rhdh-performance-olm -- sh -c 'cat /var/lib/pgsql/data/userdata/log/postgresql*.log'>postgresql.log + oc exec backstage-psql-developer-hub-0 -n $(RHDH_NAMESPACE) -- sh -c 'cat /var/lib/pgsql/data/userdata/log/postgresql*.log'>postgresql.log endif @echo "All done!!!" From 788ca9a7abba96df4509880f38c0d17a7762932c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Mac=C3=ADk?= Date: Sat, 20 Jul 2024 09:52:33 +0200 Subject: [PATCH 3/3] fix(RHIDP-2178): restart RHDH after DB is restarted with psql_debug --- ci-scripts/rhdh-setup/deploy.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/ci-scripts/rhdh-setup/deploy.sh b/ci-scripts/rhdh-setup/deploy.sh index 2329bc4..0e9e740 100755 --- a/ci-scripts/rhdh-setup/deploy.sh +++ b/ci-scripts/rhdh-setup/deploy.sh @@ -293,18 +293,27 @@ install_rhdh_with_olm() { # shellcheck disable=SC2016,SC1001,SC2086 psql_debug() { if [ "$INSTALL_METHOD" == "helm" ]; then - psql_db=rhdh-postgresql-primary-0 + psql_db_ss="${RHDH_HELM_RELEASE_NAME}-postgresql-primary" + psql_db="${psql_db_ss}-0" + rhdh_deployment="${RHDH_HELM_RELEASE_NAME}-developer-hub" elif [ "$INSTALL_METHOD" == "olm" ]; then - psql_db=backstage-psql-developer-hub-0 + psql_db_ss=backstage-psql-developer-hub + psql_db="${psql_db_ss}-0" + rhdh_deployment=backstage-developer-hub fi if ${PSQL_LOG}; then - oc exec "${psql_db}" -n "${RHDH_NAMESPACE}" -- sh -c "sed -i "s/^\s*#log_min_duration_statement.*/log_min_duration_statement=${LOG_MIN_DURATION_STATEMENT}/" /var/lib/pgsql/data/userdata/postgresql.conf " - oc exec "${psql_db}" -n "${RHDH_NAMESPACE}" -- sh -c "sed -i "s/^\s*#log_min_duration_sample.*/log_min_duration_sample=${LOG_MIN_DURATION_SAMPLE}/" /var/lib/pgsql/data/userdata/postgresql.conf " - oc exec "${psql_db}" -n "${RHDH_NAMESPACE}" -- sh -c "sed -i "s/^\s*#log_statement_sample_rate.*/log_statement_sample_rate=${LOG_STATEMENT_SAMPLE_RATE}/" /var/lib/pgsql/data/userdata/postgresql.conf " + $clin exec "${psql_db}" -- sh -c "sed -i "s/^\s*#log_min_duration_statement.*/log_min_duration_statement=${LOG_MIN_DURATION_STATEMENT}/" /var/lib/pgsql/data/userdata/postgresql.conf " + $clin exec "${psql_db}" -- sh -c "sed -i "s/^\s*#log_min_duration_sample.*/log_min_duration_sample=${LOG_MIN_DURATION_SAMPLE}/" /var/lib/pgsql/data/userdata/postgresql.conf " + $clin exec "${psql_db}" -- sh -c "sed -i "s/^\s*#log_statement_sample_rate.*/log_statement_sample_rate=${LOG_STATEMENT_SAMPLE_RATE}/" /var/lib/pgsql/data/userdata/postgresql.conf " fi - oc exec "${psql_db}" -n "${RHDH_NAMESPACE}" -- sh -c 'pg_ctl -D $PGDATA restart -mf' -} + echo "Restarting RHDH DB..." + $clin exec "${psql_db}" -- sh -c 'pg_ctl -D $PGDATA restart -mf' + wait_to_start statefulset "$psql_db_ss" 300 300 + echo "Restarting RHDH..." + $clin rollout restart deployment/"$rhdh_deployment" + wait_to_start deployment "$rhdh_deployment" 300 300 +} setup_monitoring() { echo "Enabling user workload monitoring" rm -f config.yaml