From 3df720eb07dbe40c247fe448bc636a7c4dadfcb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Mac=C3=ADk?= Date: Thu, 28 Nov 2024 08:27:04 +0100 Subject: [PATCH] fix: fix high concurrency and Helm install (#127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Pavel Macík --- ci-scripts/rhdh-setup/create_resource.sh | 2 +- ci-scripts/rhdh-setup/deploy.sh | 5 +++-- .../helm/chart-values.image-override.yaml | 3 ++- .../template/backstage/helm/chart-values.yaml | 3 ++- .../backstage/helm/extravolume-patch-1.4.yaml | 5 ----- ci-scripts/runs-to-csv.sh | 2 ++ ci-scripts/scalability/test-scalability.sh | 2 +- ci-scripts/test.sh | 7 ++++--- config/locust-k8s-operator.values.yaml | 6 +++--- requirements.txt | 2 +- scenarios/mvp-1dot1.py | 15 ++++++++------- scenarios/mvp.py | 17 ++++++++++------- 12 files changed, 37 insertions(+), 32 deletions(-) diff --git a/ci-scripts/rhdh-setup/create_resource.sh b/ci-scripts/rhdh-setup/create_resource.sh index bd3f7f0..a5c75c2 100755 --- a/ci-scripts/rhdh-setup/create_resource.sh +++ b/ci-scripts/rhdh-setup/create_resource.sh @@ -43,7 +43,7 @@ backstage_url() { } if [ ! -f "$f" ]; then if [ "$RHDH_INSTALL_METHOD" == "helm" ]; then - rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}" + rhdh_route="$(oc -n "${RHDH_NAMESPACE}" get routes -l app.kubernetes.io/instance="${RHDH_HELM_RELEASE_NAME}" -o jsonpath='{.items[0].metadata.name}')" else if [ "$AUTH_PROVIDER" == "keycloak" ]; then rhdh_route="rhdh" diff --git a/ci-scripts/rhdh-setup/deploy.sh b/ci-scripts/rhdh-setup/deploy.sh index a737cff..2dabd77 100755 --- a/ci-scripts/rhdh-setup/deploy.sh +++ b/ci-scripts/rhdh-setup/deploy.sh @@ -270,6 +270,8 @@ backstage_install() { return 1 fi date --utc -Ins >"${TMP_DIR}/populate-before" + # shellcheck disable=SC2064 + trap "date --utc -Ins >'${TMP_DIR}/populate-after'" EXIT if ${RHDH_METRIC}; then log_info "Setting up RHDH metrics" if [ "${AUTH_PROVIDER}" == "keycloak" ]; then @@ -279,7 +281,7 @@ backstage_install() { fi RHIDP-4936_RHIDP-4937_workaround # TODO: remove once https://issues.redhat.com/browse/RHIDP-4936 and https://issues.redhat.com/browse/RHIDP-4937 are fixed log_info "RHDH Installed, waiting for the catalog to be populated" - timeout=300 + timeout=600 timeout_timestamp=$(date -d "$timeout seconds" "+%s") last_count=-1 for entity_type in Component Api; do @@ -315,7 +317,6 @@ backstage_install() { sleep 10s done done - date --utc -Ins >"${TMP_DIR}/populate-after" } # shellcheck disable=SC2016,SC1004 diff --git a/ci-scripts/rhdh-setup/template/backstage/helm/chart-values.image-override.yaml b/ci-scripts/rhdh-setup/template/backstage/helm/chart-values.image-override.yaml index 482bd3f..4dcb875 100644 --- a/ci-scripts/rhdh-setup/template/backstage/helm/chart-values.image-override.yaml +++ b/ci-scripts/rhdh-setup/template/backstage/helm/chart-values.image-override.yaml @@ -1,5 +1,6 @@ global: clusterRouterBase: ${OPENSHIFT_APP_DOMAIN} + host: "${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}" imagePullSecrets: - rhdh-pull-secret dynamic: @@ -15,7 +16,7 @@ global: disabled: false route: enabled: true - host: "{{ .Values.global.host }}" + host: "${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}" path: / tls: enabled: true diff --git a/ci-scripts/rhdh-setup/template/backstage/helm/chart-values.yaml b/ci-scripts/rhdh-setup/template/backstage/helm/chart-values.yaml index 141e422..3ae4330 100644 --- a/ci-scripts/rhdh-setup/template/backstage/helm/chart-values.yaml +++ b/ci-scripts/rhdh-setup/template/backstage/helm/chart-values.yaml @@ -1,5 +1,6 @@ global: clusterRouterBase: ${OPENSHIFT_APP_DOMAIN} + host: "${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}" imagePullSecrets: - rhdh-pull-secret dynamic: @@ -15,7 +16,7 @@ global: disabled: false route: enabled: true - host: "{{ .Values.global.host }}" + host: "${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}" path: / tls: enabled: true diff --git a/ci-scripts/rhdh-setup/template/backstage/helm/extravolume-patch-1.4.yaml b/ci-scripts/rhdh-setup/template/backstage/helm/extravolume-patch-1.4.yaml index 47cfbbd..2d43dc8 100644 --- a/ci-scripts/rhdh-setup/template/backstage/helm/extravolume-patch-1.4.yaml +++ b/ci-scripts/rhdh-setup/template/backstage/helm/extravolume-patch-1.4.yaml @@ -2,8 +2,6 @@ extraVolumeMounts: - name: dynamic-plugins-root mountPath: /opt/app-root/src/dynamic-plugins-root - - name: audit-log-data - mountPath: /var/log/redhat-developer-hub/audit - name: rbac-policy mountPath: /opt/app-root/src/rbac - name: techdocs-data @@ -12,9 +10,6 @@ extraVolumes: - name: dynamic-plugins-root persistentVolumeClaim: claimName: '{{ printf "%s-dynamic-plugins-root" .Release.Name }}' - - name: audit-log-data - persistentVolumeClaim: - claimName: '{{ printf "%s-audit-log" .Release.Name }}' - name: dynamic-plugins configMap: defaultMode: 420 diff --git a/ci-scripts/runs-to-csv.sh b/ci-scripts/runs-to-csv.sh index 74954f6..e9630ba 100755 --- a/ci-scripts/runs-to-csv.sh +++ b/ci-scripts/runs-to-csv.sh @@ -22,6 +22,7 @@ Started,\ Ended,\ Duration,\ Scenario,\ +ActiveUsers,\ USERS,\ SPAWN_RATE,\ WORKERS,\ @@ -91,6 +92,7 @@ find "${1:-.}" -name benchmark.json -print0 | while IFS= read -r -d '' filename; .measurements.timings.benchmark.ended, .measurements.timings.benchmark.duration, .metadata.scenario.name, + .results.locust_users.max, .metadata.env.USERS, .metadata.env.SPAWN_RATE, .metadata.env.WORKERS, diff --git a/ci-scripts/scalability/test-scalability.sh b/ci-scripts/scalability/test-scalability.sh index 1a534a9..da549cd 100755 --- a/ci-scripts/scalability/test-scalability.sh +++ b/ci-scripts/scalability/test-scalability.sh @@ -47,7 +47,7 @@ echo wait_for_indexing() { COOKIE="$TMP_DIR/cookie.jar" if [ "$INSTALL_METHOD" == "helm" ]; then - rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}" + rhdh_route="$(oc -n "${RHDH_NAMESPACE}" get routes -l app.kubernetes.io/instance="${RHDH_HELM_RELEASE_NAME}" -o jsonpath='{.items[0].metadata.name}')" else if [ "$AUTH_PROVIDER" == "keycloak" ]; then rhdh_route="rhdh" diff --git a/ci-scripts/test.sh b/ci-scripts/test.sh index e4843f6..5e6c054 100755 --- a/ci-scripts/test.sh +++ b/ci-scripts/test.sh @@ -10,8 +10,9 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck disable=SC1090,SC1091 source "$(readlink -m "$SCRIPT_DIR"/../test.env)" -export SCENARIO RHDH_INSTALL_METHOD AUTH_PROVIDER +export SCENARIO RHDH_INSTALL_METHOD AUTH_PROVIDER RHDH_NAMESPACE +RHDH_NAMESPACE=${RHDH_NAMESPACE:-rhdh-performance} RHDH_INSTALL_METHOD=${RHDH_INSTALL_METHOD:-helm} AUTH_PROVIDER=${AUTH_PROVIDER:-keycloak} @@ -28,13 +29,13 @@ elif [ "$RHDH_INSTALL_METHOD" == "helm" ]; then RHDH_HELM_RELEASE_NAME=${RHDH_HELM_RELEASE_NAME:-rhdh} RHDH_HELM_CHART=${RHDH_HELM_CHART:-redhat-developer-hub} - rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}" + rhdh_route="$(oc -n "${RHDH_NAMESPACE}" get routes -l app.kubernetes.io/instance="${RHDH_HELM_RELEASE_NAME}" -o jsonpath='{.items[0].metadata.name}')" else echo "Invalid RHDH install method: $RHDH_INSTALL_METHOD" exit 1 fi export HOST -HOST="https://$(oc get routes "$rhdh_route" -n "${RHDH_NAMESPACE:-rhdh-performance}" -o jsonpath='{.spec.host}')" +HOST="https://$(oc get routes "$rhdh_route" -n "${RHDH_NAMESPACE}" -o jsonpath='{.spec.host}')" # end-of testing env ARTIFACT_DIR=$(readlink -m "${ARTIFACT_DIR:-.artifacts}") diff --git a/config/locust-k8s-operator.values.yaml b/config/locust-k8s-operator.values.yaml index 7827c79..e73e563 100644 --- a/config/locust-k8s-operator.values.yaml +++ b/config/locust-k8s-operator.values.yaml @@ -11,10 +11,10 @@ config: loadGenerationPods: resource: cpuRequest: 250m - memRequest: 128Mi + memRequest: 256Mi ephemeralRequest: 30Mi - cpuLimit: 2000m - memLimit: 2Gi + cpuLimit: 250m + memLimit: 256Mi ephemeralLimit: 100Mi metricsExporter: image: "quay.io/backstage-performance/locust_exporter:latest" diff --git a/requirements.txt b/requirements.txt index e4efa64..b47c7d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -https://files.pythonhosted.org/packages/54/7d/a3ec175a32df99968cec4909b69b1c05cb9367f55c058c46fb7bfe4625aa/locust-2.18.1-py3-none-any.whl \ No newline at end of file +locust==2.32.2 diff --git a/scenarios/mvp-1dot1.py b/scenarios/mvp-1dot1.py index 4e50eb6..48dda7a 100644 --- a/scenarios/mvp-1dot1.py +++ b/scenarios/mvp-1dot1.py @@ -115,16 +115,14 @@ def on_test_start(environment, **_kwargs): worker_count = environment.runner.worker_count chunk_size = int(len(users) / worker_count) + chunk_leftover = int(len(users) % worker_count) for i, worker in enumerate(environment.runner.clients): start_index = i * chunk_size - - if i + 1 < worker_count: - end_index = start_index + chunk_size - else: - end_index = len(users) - + end_index = start_index + chunk_size data = users[start_index:end_index] + if chunk_leftover > 0 and chunk_leftover > i: + data.append(users[worker_count * chunk_size + i]) environment.runner.send_message("test_users", data, worker) @@ -173,7 +171,10 @@ def __init__(self, parent): super().__init__(parent) self.HEADER = '' if self.environment.parsed_options.keycloak_host: - self.USERNAME = usernames.pop() + if len(usernames) > 0: + self.USERNAME = usernames.pop() + else: + self.USERNAME = "t1" kc_host = self.environment.parsed_options.keycloak_host self.KEYCLOAK_URL = f'https://{kc_host}/auth' bs_host = self.environment.host diff --git a/scenarios/mvp.py b/scenarios/mvp.py index 99ef35c..159103f 100644 --- a/scenarios/mvp.py +++ b/scenarios/mvp.py @@ -104,6 +104,7 @@ def get_entities_by_query_params(kind=None, limit=0, user_ref=None, group_ref=No def setup_test_users(environment, msg, **kwargs): # Fired when the worker receives a message of type 'test_users' usernames.extend(map(lambda u: u, msg.data)) + print(f"Usernames: {usernames}") @events.init.add_listener @@ -123,16 +124,15 @@ def on_test_start(environment, **_kwargs): worker_count = environment.runner.worker_count chunk_size = int(len(users) / worker_count) + chunk_leftover = int(len(users) % worker_count) for i, worker in enumerate(environment.runner.clients): start_index = i * chunk_size - - if i + 1 < worker_count: - end_index = start_index + chunk_size - else: - end_index = len(users) - + end_index = start_index + chunk_size data = users[start_index:end_index] + if chunk_leftover > 0 and chunk_leftover > i: + data.append(users[worker_count * chunk_size + i]) + print(f"Setting up test users {data}...") environment.runner.send_message("test_users", data, worker) @@ -200,7 +200,10 @@ def __init__(self, parent): super().__init__(parent) self.HEADER = '' if self.environment.parsed_options.keycloak_host: - self.USERNAME = usernames.pop() + if len(usernames) > 0: + self.USERNAME = usernames.pop() + else: + self.USERNAME = "t1" kc_host = self.environment.parsed_options.keycloak_host self.KEYCLOAK_URL = f'https://{kc_host}/auth' bs_host = self.environment.host