fix: fix high concurrency and Helm install (#127)

Signed-off-by: Pavel Macík <[email protected]>

ci-scripts/rhdh-setup/create_resource.sh

@@ -43,7 +43,7 @@ backstage_url() {
   }
   if [ ! -f "$f" ]; then
     if [ "$RHDH_INSTALL_METHOD" == "helm" ]; then
-      rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}"
+      rhdh_route="$(oc -n "${RHDH_NAMESPACE}" get routes -l app.kubernetes.io/instance="${RHDH_HELM_RELEASE_NAME}" -o jsonpath='{.items[0].metadata.name}')"
     else
       if [ "$AUTH_PROVIDER" == "keycloak" ]; then
         rhdh_route="rhdh"

ci-scripts/rhdh-setup/deploy.sh

@@ -270,6 +270,8 @@ backstage_install() {
         return 1
     fi
     date --utc -Ins >"${TMP_DIR}/populate-before"
+    # shellcheck disable=SC2064
+    trap "date --utc -Ins >'${TMP_DIR}/populate-after'" EXIT
     if ${RHDH_METRIC}; then
         log_info "Setting up RHDH metrics"
         if [ "${AUTH_PROVIDER}" == "keycloak" ]; then
@@ -279,7 +281,7 @@ backstage_install() {
     fi
     RHIDP-4936_RHIDP-4937_workaround # TODO: remove once https://issues.redhat.com/browse/RHIDP-4936 and https://issues.redhat.com/browse/RHIDP-4937 are fixed
     log_info "RHDH Installed, waiting for the catalog to be populated"
-    timeout=300
+    timeout=600
     timeout_timestamp=$(date -d "$timeout seconds" "+%s")
     last_count=-1
     for entity_type in Component Api; do
@@ -315,7 +317,6 @@ backstage_install() {
             sleep 10s
         done
     done
-    date --utc -Ins >"${TMP_DIR}/populate-after"
 }
 
 # shellcheck disable=SC2016,SC1004

ci-scripts/rhdh-setup/template/backstage/helm/chart-values.image-override.yaml

@@ -1,5 +1,6 @@
 global:
   clusterRouterBase: ${OPENSHIFT_APP_DOMAIN}
+  host: "${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}"
   imagePullSecrets:
     - rhdh-pull-secret
   dynamic:
@@ -15,7 +16,7 @@ global:
         disabled: false
 route:
   enabled: true
-  host: "{{ .Values.global.host }}"
+  host: "${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}"
   path: /
   tls:
     enabled: true

ci-scripts/rhdh-setup/template/backstage/helm/chart-values.yaml

@@ -1,5 +1,6 @@
 global:
   clusterRouterBase: ${OPENSHIFT_APP_DOMAIN}
+  host: "${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}"
   imagePullSecrets:
     - rhdh-pull-secret
   dynamic:
@@ -15,7 +16,7 @@ global:
         disabled: false
 route:
   enabled: true
-  host: "{{ .Values.global.host }}"
+  host: "${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}"
   path: /
   tls:
     enabled: true

ci-scripts/rhdh-setup/template/backstage/helm/extravolume-patch-1.4.yaml

@@ -2,8 +2,6 @@
 extraVolumeMounts:
   - name: dynamic-plugins-root
     mountPath: /opt/app-root/src/dynamic-plugins-root
-  - name: audit-log-data
-    mountPath: /var/log/redhat-developer-hub/audit
   - name: rbac-policy
     mountPath: /opt/app-root/src/rbac
   - name: techdocs-data
@@ -12,9 +10,6 @@ extraVolumes:
   - name: dynamic-plugins-root
     persistentVolumeClaim:
       claimName: '{{ printf "%s-dynamic-plugins-root" .Release.Name }}'
-  - name: audit-log-data
-    persistentVolumeClaim:
-      claimName: '{{ printf "%s-audit-log" .Release.Name }}'
   - name: dynamic-plugins
     configMap:
       defaultMode: 420

ci-scripts/runs-to-csv.sh

@@ -22,6 +22,7 @@ Started,\
 Ended,\
 Duration,\
 Scenario,\
+ActiveUsers,\
 USERS,\
 SPAWN_RATE,\
 WORKERS,\
@@ -91,6 +92,7 @@ find "${1:-.}" -name benchmark.json -print0 | while IFS= read -r -d '' filename;
         .measurements.timings.benchmark.ended,
         .measurements.timings.benchmark.duration,
         .metadata.scenario.name,
+        .results.locust_users.max,
         .metadata.env.USERS,
         .metadata.env.SPAWN_RATE,
         .metadata.env.WORKERS,

ci-scripts/scalability/test-scalability.sh

@@ -47,7 +47,7 @@ echo
 wait_for_indexing() {
     COOKIE="$TMP_DIR/cookie.jar"
     if [ "$INSTALL_METHOD" == "helm" ]; then
-        rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}"
+        rhdh_route="$(oc -n "${RHDH_NAMESPACE}" get routes -l app.kubernetes.io/instance="${RHDH_HELM_RELEASE_NAME}" -o jsonpath='{.items[0].metadata.name}')"
     else
         if [ "$AUTH_PROVIDER" == "keycloak" ]; then
             rhdh_route="rhdh"

ci-scripts/test.sh

@@ -10,8 +10,9 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck disable=SC1090,SC1091
 source "$(readlink -m "$SCRIPT_DIR"/../test.env)"
 
-export SCENARIO RHDH_INSTALL_METHOD AUTH_PROVIDER
+export SCENARIO RHDH_INSTALL_METHOD AUTH_PROVIDER RHDH_NAMESPACE
 
+RHDH_NAMESPACE=${RHDH_NAMESPACE:-rhdh-performance}
 RHDH_INSTALL_METHOD=${RHDH_INSTALL_METHOD:-helm}
 AUTH_PROVIDER=${AUTH_PROVIDER:-keycloak}
 
@@ -28,13 +29,13 @@ elif [ "$RHDH_INSTALL_METHOD" == "helm" ]; then
     RHDH_HELM_RELEASE_NAME=${RHDH_HELM_RELEASE_NAME:-rhdh}
     RHDH_HELM_CHART=${RHDH_HELM_CHART:-redhat-developer-hub}
 
-    rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}"
+    rhdh_route="$(oc -n "${RHDH_NAMESPACE}" get routes -l app.kubernetes.io/instance="${RHDH_HELM_RELEASE_NAME}" -o jsonpath='{.items[0].metadata.name}')"
 else
     echo "Invalid RHDH install method: $RHDH_INSTALL_METHOD"
     exit 1
 fi
 export HOST
-HOST="https://$(oc get routes "$rhdh_route" -n "${RHDH_NAMESPACE:-rhdh-performance}" -o jsonpath='{.spec.host}')"
+HOST="https://$(oc get routes "$rhdh_route" -n "${RHDH_NAMESPACE}" -o jsonpath='{.spec.host}')"
 # end-of testing env
 
 ARTIFACT_DIR=$(readlink -m "${ARTIFACT_DIR:-.artifacts}")

config/locust-k8s-operator.values.yaml

@@ -11,10 +11,10 @@ config:
   loadGenerationPods:
     resource:
       cpuRequest: 250m
-      memRequest: 128Mi
+      memRequest: 256Mi
       ephemeralRequest: 30Mi
-      cpuLimit: 2000m
-      memLimit: 2Gi
+      cpuLimit: 250m
+      memLimit: 256Mi
       ephemeralLimit: 100Mi
     metricsExporter:
       image: "quay.io/backstage-performance/locust_exporter:latest"

requirements.txt

@@ -1 +1 @@
-https://files.pythonhosted.org/packages/54/7d/a3ec175a32df99968cec4909b69b1c05cb9367f55c058c46fb7bfe4625aa/locust-2.18.1-py3-none-any.whl
+locust==2.32.2

scenarios/mvp-1dot1.py

@@ -115,16 +115,14 @@ def on_test_start(environment, **_kwargs):
 
         worker_count = environment.runner.worker_count
         chunk_size = int(len(users) / worker_count)
+        chunk_leftover = int(len(users) % worker_count)
 
         for i, worker in enumerate(environment.runner.clients):
             start_index = i * chunk_size
-
-            if i + 1 < worker_count:
-                end_index = start_index + chunk_size
-            else:
-                end_index = len(users)
-
+            end_index = start_index + chunk_size
             data = users[start_index:end_index]
+            if chunk_leftover > 0 and chunk_leftover > i:
+                data.append(users[worker_count * chunk_size + i])
             environment.runner.send_message("test_users", data, worker)
 
 
@@ -173,7 +171,10 @@ def __init__(self, parent):
         super().__init__(parent)
         self.HEADER = ''
         if self.environment.parsed_options.keycloak_host:
-            self.USERNAME = usernames.pop()
+            if len(usernames) > 0:
+                self.USERNAME = usernames.pop()
+            else:
+                self.USERNAME = "t1"
             kc_host = self.environment.parsed_options.keycloak_host
             self.KEYCLOAK_URL = f'https://{kc_host}/auth'
             bs_host = self.environment.host

scenarios/mvp.py

@@ -104,6 +104,7 @@ def get_entities_by_query_params(kind=None, limit=0, user_ref=None, group_ref=No
 def setup_test_users(environment, msg, **kwargs):
     # Fired when the worker receives a message of type 'test_users'
     usernames.extend(map(lambda u: u, msg.data))
+    print(f"Usernames: {usernames}")
 
 
 @events.init.add_listener
@@ -123,16 +124,15 @@ def on_test_start(environment, **_kwargs):
 
         worker_count = environment.runner.worker_count
         chunk_size = int(len(users) / worker_count)
+        chunk_leftover = int(len(users) % worker_count)
 
         for i, worker in enumerate(environment.runner.clients):
             start_index = i * chunk_size
-
-            if i + 1 < worker_count:
-                end_index = start_index + chunk_size
-            else:
-                end_index = len(users)
-
+            end_index = start_index + chunk_size
             data = users[start_index:end_index]
+            if chunk_leftover > 0 and chunk_leftover > i:
+                data.append(users[worker_count * chunk_size + i])
+            print(f"Setting up test users {data}...")
             environment.runner.send_message("test_users", data, worker)
 
 
@@ -200,7 +200,10 @@ def __init__(self, parent):
         super().__init__(parent)
         self.HEADER = ''
         if self.environment.parsed_options.keycloak_host:
-            self.USERNAME = usernames.pop()
+            if len(usernames) > 0:
+                self.USERNAME = usernames.pop()
+            else:
+                self.USERNAME = "t1"
             kc_host = self.environment.parsed_options.keycloak_host
             self.KEYCLOAK_URL = f'https://{kc_host}/auth'
             bs_host = self.environment.host