Add comparison script and readme updates

Signed-off-by: David Martin <[email protected]>
Kuadrant · Oct 22, 2024 · 7d12747 · 7d12747
1 parent dda22bd
commit 7d12747
Show file tree

Hide file tree

Showing 2 changed files with 350 additions and 0 deletions.
diff --git a/scale_test/compare.sh b/scale_test/compare.sh
@@ -0,0 +1,291 @@
+#!/usr/bin/env bash
+
+# Elasticsearch parameters
+ES_INDEX="kube-burner"
+ES_URL="$ES_SERVER/$ES_INDEX/_search"
+
+# If UUID1 and UUID2 are not set, get the most recent two jobSummary entries
+if [ -z "$UUID1" ] || [ -z "$UUID2" ]; then
+  # Get the most recent two jobSummary entries
+  curl -k -s -X GET "$ES_URL" \
+    -H 'Content-Type: application/json' \
+    -d '{
+      "size": 2,
+      "query": {
+        "bool": {
+          "must": [
+            { "term": { "metricName.keyword": "jobSummary" } },
+            { "term": { "jobConfig.name.keyword": "scale-test-main" } }
+          ]
+        }
+      },
+      "sort": [
+        { "timestamp": { "order": "desc" } }
+      ],
+      "_source": ["uuid", "elapsedTime", "timestamp"]
+    }' > jobSummary.json
+
+  # Extract the uuids, elapsedTime values, and timestamps
+  UUID1=$(jq -r '.hits.hits[1]._source.uuid' jobSummary.json)
+  UUID2=$(jq -r '.hits.hits[0]._source.uuid' jobSummary.json)
+
+  ELAPSED1=$(jq -r '.hits.hits[1]._source.elapsedTime' jobSummary.json)
+  ELAPSED2=$(jq -r '.hits.hits[0]._source.elapsedTime' jobSummary.json)
+
+  TIMESTAMP1=$(jq -r '.hits.hits[1]._source.timestamp' jobSummary.json)
+  TIMESTAMP2=$(jq -r '.hits.hits[0]._source.timestamp' jobSummary.json)
+else
+  # Use the provided UUID1 and UUID2 to get elapsedTime and timestamp
+  # Fetch ELAPSED1 and TIMESTAMP1 for UUID1
+  response1=$(curl -k -s -X GET "$ES_URL" \
+    -H 'Content-Type: application/json' \
+    -d "{
+      \"size\": 1,
+      \"query\": {
+        \"bool\": {
+          \"must\": [
+            { \"term\": { \"metricName.keyword\": \"jobSummary\" } },
+            { \"term\": { \"uuid.keyword\": \"$UUID1\" } },
+            { \"term\": { \"jobConfig.name.keyword\": \"scale-test-main\" } }
+          ]
+        }
+      },
+      \"_source\": [\"elapsedTime\", \"timestamp\"]
+    }")
+  ELAPSED1=$(echo "$response1" | jq -r '.hits.hits[0]._source.elapsedTime')
+  TIMESTAMP1=$(echo "$response1" | jq -r '.hits.hits[0]._source.timestamp')
+
+  # Fetch ELAPSED2 and TIMESTAMP2 for UUID2
+  response2=$(curl -k -s -X GET "$ES_URL" \
+    -H 'Content-Type: application/json' \
+    -d "{
+      \"size\": 1,
+      \"query\": {
+        \"bool\": {
+          \"must\": [
+            { \"term\": { \"metricName.keyword\": \"jobSummary\" } },
+            { \"term\": { \"uuid.keyword\": \"$UUID2\" } },
+            { \"term\": { \"jobConfig.name.keyword\": \"scale-test-main\" } }
+          ]
+        }
+      },
+      \"_source\": [\"elapsedTime\", \"timestamp\"]
+    }")
+  ELAPSED2=$(echo "$response2" | jq -r '.hits.hits[0]._source.elapsedTime')
+  TIMESTAMP2=$(echo "$response2" | jq -r '.hits.hits[0]._source.timestamp')
+fi
+
+# Shorten UUIDs for column headers
+UUID_SHORT1="${UUID1:0:8}"
+UUID_SHORT2="${UUID2:0:8}"
+
+# Function to get per-namespace CPU values for a given uuid
+get_namespace_metric_values() {
+  local uuid="$1"
+  local metricName="$2"
+
+  curl -k -s -X GET "$ES_URL" \
+    -H 'Content-Type: application/json' \
+    -d "{
+      \"size\": 0,
+      \"query\": {
+        \"bool\": {
+          \"must\": [
+            { \"term\": { \"metricName.keyword\": \"$metricName\" } },
+            { \"term\": { \"uuid.keyword\": \"$uuid\" } },
+            { \"term\": { \"jobName.keyword\": \"scale-test-main\" } }
+          ]
+        }
+      },
+      \"aggs\": {
+        \"namespaces\": {
+          \"terms\": {
+            \"field\": \"labels.namespace.keyword\",
+            \"size\": 1000
+          },
+          \"aggs\": {
+            \"avg_value\": {
+              \"avg\": {
+                \"field\": \"value\"
+              }
+            }
+          }
+        }
+      }
+    }" | jq -r '.aggregations.namespaces.buckets[] | [ .key, (.avg_value.value | tostring) ] | @tsv'
+}
+
+# Function to get per-controller reconcile time values for a given uuid
+get_controller_reconcile() {
+  local uuid="$1"
+
+  curl -k -s -X GET "$ES_URL" \
+    -H 'Content-Type: application/json' \
+    -d "{
+      \"size\": 50,
+      \"query\": {
+        \"bool\": {
+          \"must\": [
+            { \"term\": { \"metricName.keyword\": \"Controller99thReconcile\" } },
+            { \"term\": { \"uuid.keyword\": \"$uuid\" } },
+            { \"term\": { \"jobName.keyword\": \"scale-test-main\" } }
+          ]
+        }
+      },
+      \"_source\": [\"value\", \"labels.controller\"],
+      \"sort\": [
+        { \"timestamp\": { \"order\": \"desc\" } }
+      ]
+    }" | jq -r '.hits.hits[] | [ .["_source"]["labels"]["controller"], .["_source"]["value"] ] | @tsv'
+}
+
+# Get per-namespace CPU values
+declare -A CPU1_VALUES
+declare -A CPU2_VALUES
+while IFS=$'\t' read -r namespace value; do
+  if [ -z "${CPU1_VALUES["$namespace"]}" ]; then
+    CPU1_VALUES["$namespace"]=$value
+  fi
+done < <(get_namespace_metric_values "$UUID1" "namespaceCPU")
+
+while IFS=$'\t' read -r namespace value; do
+  if [ -z "${CPU2_VALUES["$namespace"]}" ]; then
+    CPU2_VALUES["$namespace"]=$value
+  fi
+done < <(get_namespace_metric_values "$UUID2" "namespaceCPU")
+
+ALL_CPU_NAMESPACES=()
+for namespace in "${!CPU1_VALUES[@]}"; do
+  ALL_CPU_NAMESPACES+=("$namespace")
+done
+for namespace in "${!CPU2_VALUES[@]}"; do
+  if [[ ! " ${ALL_CPU_NAMESPACES[@]} " =~ " ${namespace} " ]]; then
+    ALL_CPU_NAMESPACES+=("$namespace")
+  fi
+done
+
+# Get per-namespace Memory values
+declare -A MEM1_VALUES
+declare -A MEM2_VALUES
+while IFS=$'\t' read -r namespace value; do
+  if [ -z "${MEM1_VALUES["$namespace"]}" ]; then
+    MEM1_VALUES["$namespace"]=$value
+  fi
+done < <(get_namespace_metric_values "$UUID1" "namespaceMemory")
+
+while IFS=$'\t' read -r namespace value; do
+  if [ -z "${MEM2_VALUES["$namespace"]}" ]; then
+    MEM2_VALUES["$namespace"]=$value
+  fi
+done < <(get_namespace_metric_values "$UUID2" "namespaceMemory")
+
+ALL_MEM_NAMESPACES=()
+for namespace in "${!MEM1_VALUES[@]}"; do
+  ALL_MEM_NAMESPACES+=("$namespace")
+done
+for namespace in "${!MEM2_VALUES[@]}"; do
+  if [[ ! " ${ALL_MEM_NAMESPACES[@]} " =~ " ${namespace} " ]]; then
+    ALL_MEM_NAMESPACES+=("$namespace")
+  fi
+done
+
+# Get per-controller reconcile time values
+declare -A RECONCILE1_VALUES
+declare -A RECONCILE2_VALUES
+while IFS=$'\t' read -r namespace value; do
+  if [ -z "${RECONCILE1_VALUES["$namespace"]}" ]; then
+    RECONCILE1_VALUES["$namespace"]=$value
+  fi
+done < <(get_controller_reconcile "$UUID1")
+
+while IFS=$'\t' read -r namespace value; do
+  if [ -z "${RECONCILE2_VALUES["$namespace"]}" ]; then
+    RECONCILE2_VALUES["$namespace"]=$value
+  fi
+done < <(get_controller_reconcile "$UUID2")
+
+ALL_RECONCILE_CONTROLLERS=()
+for namespace in "${!RECONCILE1_VALUES[@]}"; do
+  ALL_RECONCILE_CONTROLLERS+=("$namespace")
+done
+for namespace in "${!RECONCILE2_VALUES[@]}"; do
+  if [[ ! " ${ALL_RECONCILE_CONTROLLERS[@]} " =~ " ${namespace} " ]]; then
+    ALL_RECONCILE_CONTROLLERS+=("$namespace")
+  fi
+done
+
+# Compare the values and compute differences
+compare_values() {
+  local val1="$1"
+  local val2="$2"
+  local format="$3"
+  local diff
+  local sign
+
+  diff=$(echo "$val2 - $val1" | bc -l)
+  if (( $(echo "$diff < 0" | bc -l) )); then
+    sign="-"
+    diff=$(echo "$val1 - $val2" | bc -l)
+  else
+    sign="+"
+  fi
+
+  printf "%s${format}" "$sign" "$diff"
+}
+
+ELAPSED_DIFF=$(compare_values "$ELAPSED1" "$ELAPSED2" "%.0f")
+
+# Output the report
+printf "%-25s %-27s %-27s %-20s\n" "Metric" "UUID 1 ($UUID_SHORT1)" "UUID 2 ($UUID_SHORT2)" "Diff"
+printf "%-25s %-27s %-27s %-20s\n" "Timestamp" "$TIMESTAMP1" "$TIMESTAMP2" ""
+printf "%-25s %-27s %-27s %-20s\n" "Elapsed Time" "$ELAPSED1" "$ELAPSED2" "$ELAPSED_DIFF"
+printf "Namespace CPU\n"
+for namespace in "${ALL_CPU_NAMESPACES[@]}"; do
+  value1="${CPU1_VALUES[$namespace]}"
+  value2="${CPU2_VALUES[$namespace]}"
+
+  if [ -z "$value1" ]; then value1=0; fi
+  if [ -z "$value2" ]; then value2=0; fi
+  diff=$(compare_values "$value1" "$value2" "%.4f")
+
+  value1_formatted=$(printf "%.4f" "$value1")
+  value2_formatted=$(printf "%.4f" "$value2")
+
+  printf "%-25s %-27s %-27s %-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$diff"
+done
+printf "Namespace Memory (MB)\n"
+for namespace in "${ALL_MEM_NAMESPACES[@]}"; do
+  value1="${MEM1_VALUES[$namespace]}"
+  value2="${MEM2_VALUES[$namespace]}"
+
+  if [ -z "$value1" ]; then value1=0; fi
+  if [ -z "$value2" ]; then value2=0; fi
+  diff=$(echo "$value2 - $value1" | bc -l)
+  sign=""
+  if (( $(echo "$diff < 0" | bc -l) )); then
+    sign="-"
+    diff=$(echo "$value1 - $value2" | bc -l)
+  else
+    sign="+"
+  fi
+  diff=$(echo "scale=2; $diff / 1048576" | bc)
+
+  value1_formatted=$(echo "scale=2; $value1 / 1048576" | bc)
+  value2_formatted=$(echo "scale=2; $value2 / 1048576" | bc)
+
+  printf "%-25s %-27s %-27s %s%-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$sign" "$diff"
+done
+printf "Controller 99th Reconcile (s)\n"
+for namespace in "${ALL_RECONCILE_CONTROLLERS[@]}"; do
+  value1="${RECONCILE1_VALUES[$namespace]}"
+  value2="${RECONCILE2_VALUES[$namespace]}"
+
+  if [ -z "$value1" ]; then value1=0; fi
+  if [ -z "$value2" ]; then value2=0; fi
+  diff=$(compare_values "$value1" "$value2" "%.4f")
+
+  value1_formatted=$(printf "%.4f" "$value1")
+  value2_formatted=$(printf "%.4f" "$value2")
+
+  printf "%-25s %-27s %-27s %-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$diff"
+done
diff --git a/scale_test/readme.md b/scale_test/readme.md
@@ -179,3 +179,62 @@ Some common transforms are:
 * "Sort by"
 
 Inspecting some of the existing panels in the Kuadrant dashboard and kube-burner dashboards at https://github.com/kube-burner/kube-burner/tree/main/examples/grafana-dashboards can give inspiration as well.
+
+## Performance Comparison Script
+
+The `./compare.sh` script allows you to compare key performance metrics between two test runs. It fetches metrics from an Elasticsearch instance and generates a report highlighting the differences in performance between the two specified runs, similar to the comparison Grafana dashboard. Values are calculated based on the average of most recent values.
+
+You can specify the UUIDs of the two test runs you wish to compare by setting the `UUID1` and `UUID2` environment variables when running the script.
+
+```bash
+UUID1=<first-test-uuid> UUID2=<second-test-uuid> ./compare.sh
+```
+
+**Example**:
+
+```bash
+UUID1=926c7847-95bd-4ed2-bf80-2cad6746db9c UUID2=c1195a3d-6ceb-4468-951c-d0492ef7c79c ./compare.sh
+```
+
+### Automatic UUID Selection
+
+If you do not specify `UUID1` and `UUID2`, the script will automatically fetch the two most recent test runs from Elasticsearch and compare them.
+
+```bash
+./compare.sh
+```
+
+## Example Output
+
+```
+Metric                    UUID 1 (926c7847)           UUID 2 (c1195a3d)           Diff
+Timestamp                 2024-10-18T12:01:11.588843Z 2024-10-18T12:50:47.638731Z
+Elapsed Time              152                         220                         +68
+
+Namespace CPU (Average Values)
+Namespace                 926c7847        c1195a3d        Diff
+kuadrant-system           0.0119          0.0122          +0.0003
+scale-test-0              0.0006          0.1076          +0.1070
+gateway-system            0.0022          0.0019          -0.0003
+istio-system              0.0071          0.0317          +0.0246
+
+Namespace Memory (MB)
+Namespace                 926c7847        c1195a3d        Diff
+kuadrant-system           181.78          152.12          -29.65
+scale-test-0              168.64          267.74          +99.09
+gateway-system            40.30           39.59           -0.70
+istio-system              114.27          119.07          +4.79
+
+Controller 99th Reconcile (s)
+Controller                926c7847        c1195a3d        Diff
+dnsrecord                 5.8800          2.9600          -2.9200
+authpolicy                0.1835          0.0955          -0.0880
+dnspolicy                 0.1410          0.1423          +0.0012
+kuadrant                  0.0980          0.1475          +0.0495
+limitador                 0.0980          0.3975          +0.2995
+gateway                   0.1474          0.0885          -0.0588
+controller                3.4350          1.2150          -2.2200
+tlspolicy                 3.8300          0.0959          -3.7341
+ratelimitpolicy           0.1760          0.0902          -0.0858
+authorino                 0.0000          0.6955          +0.6955
+```