diff --git a/scale_test/compare.sh b/scale_test/compare.sh new file mode 100755 index 00000000..87ef25de --- /dev/null +++ b/scale_test/compare.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash + +# Elasticsearch parameters +ES_INDEX="kube-burner" +ES_URL="$ES_SERVER/$ES_INDEX/_search" + +# If UUID1 and UUID2 are not set, get the most recent two jobSummary entries +if [ -z "$UUID1" ] || [ -z "$UUID2" ]; then + # Get the most recent two jobSummary entries + curl -k -s -X GET "$ES_URL" \ + -H 'Content-Type: application/json' \ + -d '{ + "size": 2, + "query": { + "bool": { + "must": [ + { "term": { "metricName.keyword": "jobSummary" } }, + { "term": { "jobConfig.name.keyword": "scale-test-main" } } + ] + } + }, + "sort": [ + { "timestamp": { "order": "desc" } } + ], + "_source": ["uuid", "elapsedTime", "timestamp"] + }' > jobSummary.json + + # Extract the uuids, elapsedTime values, and timestamps + UUID1=$(jq -r '.hits.hits[1]._source.uuid' jobSummary.json) + UUID2=$(jq -r '.hits.hits[0]._source.uuid' jobSummary.json) + + ELAPSED1=$(jq -r '.hits.hits[1]._source.elapsedTime' jobSummary.json) + ELAPSED2=$(jq -r '.hits.hits[0]._source.elapsedTime' jobSummary.json) + + TIMESTAMP1=$(jq -r '.hits.hits[1]._source.timestamp' jobSummary.json) + TIMESTAMP2=$(jq -r '.hits.hits[0]._source.timestamp' jobSummary.json) +else + # Use the provided UUID1 and UUID2 to get elapsedTime and timestamp + # Fetch ELAPSED1 and TIMESTAMP1 for UUID1 + response1=$(curl -k -s -X GET "$ES_URL" \ + -H 'Content-Type: application/json' \ + -d "{ + \"size\": 1, + \"query\": { + \"bool\": { + \"must\": [ + { \"term\": { \"metricName.keyword\": \"jobSummary\" } }, + { \"term\": { \"uuid.keyword\": \"$UUID1\" } }, + { \"term\": { \"jobConfig.name.keyword\": \"scale-test-main\" } } + ] + } + }, + \"_source\": [\"elapsedTime\", \"timestamp\"] + }") + ELAPSED1=$(echo "$response1" | jq -r '.hits.hits[0]._source.elapsedTime') + TIMESTAMP1=$(echo "$response1" | jq -r '.hits.hits[0]._source.timestamp') + + # Fetch ELAPSED2 and TIMESTAMP2 for UUID2 + response2=$(curl -k -s -X GET "$ES_URL" \ + -H 'Content-Type: application/json' \ + -d "{ + \"size\": 1, + \"query\": { + \"bool\": { + \"must\": [ + { \"term\": { \"metricName.keyword\": \"jobSummary\" } }, + { \"term\": { \"uuid.keyword\": \"$UUID2\" } }, + { \"term\": { \"jobConfig.name.keyword\": \"scale-test-main\" } } + ] + } + }, + \"_source\": [\"elapsedTime\", \"timestamp\"] + }") + ELAPSED2=$(echo "$response2" | jq -r '.hits.hits[0]._source.elapsedTime') + TIMESTAMP2=$(echo "$response2" | jq -r '.hits.hits[0]._source.timestamp') +fi + +# Shorten UUIDs for column headers +UUID_SHORT1="${UUID1:0:8}" +UUID_SHORT2="${UUID2:0:8}" + +# Function to get per-namespace CPU values for a given uuid +get_namespace_metric_values() { + local uuid="$1" + local metricName="$2" + + curl -k -s -X GET "$ES_URL" \ + -H 'Content-Type: application/json' \ + -d "{ + \"size\": 0, + \"query\": { + \"bool\": { + \"must\": [ + { \"term\": { \"metricName.keyword\": \"$metricName\" } }, + { \"term\": { \"uuid.keyword\": \"$uuid\" } }, + { \"term\": { \"jobName.keyword\": \"scale-test-main\" } } + ] + } + }, + \"aggs\": { + \"namespaces\": { + \"terms\": { + \"field\": \"labels.namespace.keyword\", + \"size\": 1000 + }, + \"aggs\": { + \"avg_value\": { + \"avg\": { + \"field\": \"value\" + } + } + } + } + } + }" | jq -r '.aggregations.namespaces.buckets[] | [ .key, (.avg_value.value | tostring) ] | @tsv' +} + +# Function to get per-controller reconcile time values for a given uuid +get_controller_reconcile() { + local uuid="$1" + + curl -k -s -X GET "$ES_URL" \ + -H 'Content-Type: application/json' \ + -d "{ + \"size\": 50, + \"query\": { + \"bool\": { + \"must\": [ + { \"term\": { \"metricName.keyword\": \"Controller99thReconcile\" } }, + { \"term\": { \"uuid.keyword\": \"$uuid\" } }, + { \"term\": { \"jobName.keyword\": \"scale-test-main\" } } + ] + } + }, + \"_source\": [\"value\", \"labels.controller\"], + \"sort\": [ + { \"timestamp\": { \"order\": \"desc\" } } + ] + }" | jq -r '.hits.hits[] | [ .["_source"]["labels"]["controller"], .["_source"]["value"] ] | @tsv' +} + +# Get per-namespace CPU values +declare -A CPU1_VALUES +declare -A CPU2_VALUES +while IFS=$'\t' read -r namespace value; do + if [ -z "${CPU1_VALUES["$namespace"]}" ]; then + CPU1_VALUES["$namespace"]=$value + fi +done < <(get_namespace_metric_values "$UUID1" "namespaceCPU") + +while IFS=$'\t' read -r namespace value; do + if [ -z "${CPU2_VALUES["$namespace"]}" ]; then + CPU2_VALUES["$namespace"]=$value + fi +done < <(get_namespace_metric_values "$UUID2" "namespaceCPU") + +ALL_CPU_NAMESPACES=() +for namespace in "${!CPU1_VALUES[@]}"; do + ALL_CPU_NAMESPACES+=("$namespace") +done +for namespace in "${!CPU2_VALUES[@]}"; do + if [[ ! " ${ALL_CPU_NAMESPACES[@]} " =~ " ${namespace} " ]]; then + ALL_CPU_NAMESPACES+=("$namespace") + fi +done + +# Get per-namespace Memory values +declare -A MEM1_VALUES +declare -A MEM2_VALUES +while IFS=$'\t' read -r namespace value; do + if [ -z "${MEM1_VALUES["$namespace"]}" ]; then + MEM1_VALUES["$namespace"]=$value + fi +done < <(get_namespace_metric_values "$UUID1" "namespaceMemory") + +while IFS=$'\t' read -r namespace value; do + if [ -z "${MEM2_VALUES["$namespace"]}" ]; then + MEM2_VALUES["$namespace"]=$value + fi +done < <(get_namespace_metric_values "$UUID2" "namespaceMemory") + +ALL_MEM_NAMESPACES=() +for namespace in "${!MEM1_VALUES[@]}"; do + ALL_MEM_NAMESPACES+=("$namespace") +done +for namespace in "${!MEM2_VALUES[@]}"; do + if [[ ! " ${ALL_MEM_NAMESPACES[@]} " =~ " ${namespace} " ]]; then + ALL_MEM_NAMESPACES+=("$namespace") + fi +done + +# Get per-controller reconcile time values +declare -A RECONCILE1_VALUES +declare -A RECONCILE2_VALUES +while IFS=$'\t' read -r namespace value; do + if [ -z "${RECONCILE1_VALUES["$namespace"]}" ]; then + RECONCILE1_VALUES["$namespace"]=$value + fi +done < <(get_controller_reconcile "$UUID1") + +while IFS=$'\t' read -r namespace value; do + if [ -z "${RECONCILE2_VALUES["$namespace"]}" ]; then + RECONCILE2_VALUES["$namespace"]=$value + fi +done < <(get_controller_reconcile "$UUID2") + +ALL_RECONCILE_CONTROLLERS=() +for namespace in "${!RECONCILE1_VALUES[@]}"; do + ALL_RECONCILE_CONTROLLERS+=("$namespace") +done +for namespace in "${!RECONCILE2_VALUES[@]}"; do + if [[ ! " ${ALL_RECONCILE_CONTROLLERS[@]} " =~ " ${namespace} " ]]; then + ALL_RECONCILE_CONTROLLERS+=("$namespace") + fi +done + +# Compare the values and compute differences +compare_values() { + local val1="$1" + local val2="$2" + local format="$3" + local diff + local sign + + diff=$(echo "$val2 - $val1" | bc -l) + if (( $(echo "$diff < 0" | bc -l) )); then + sign="-" + diff=$(echo "$val1 - $val2" | bc -l) + else + sign="+" + fi + + printf "%s${format}" "$sign" "$diff" +} + +ELAPSED_DIFF=$(compare_values "$ELAPSED1" "$ELAPSED2" "%.0f") + +# Output the report +printf "%-25s %-27s %-27s %-20s\n" "Metric" "UUID 1 ($UUID_SHORT1)" "UUID 2 ($UUID_SHORT2)" "Diff" +printf "%-25s %-27s %-27s %-20s\n" "Timestamp" "$TIMESTAMP1" "$TIMESTAMP2" "" +printf "%-25s %-27s %-27s %-20s\n" "Elapsed Time" "$ELAPSED1" "$ELAPSED2" "$ELAPSED_DIFF" +printf "Namespace CPU\n" +for namespace in "${ALL_CPU_NAMESPACES[@]}"; do + value1="${CPU1_VALUES[$namespace]}" + value2="${CPU2_VALUES[$namespace]}" + + if [ -z "$value1" ]; then value1=0; fi + if [ -z "$value2" ]; then value2=0; fi + diff=$(compare_values "$value1" "$value2" "%.4f") + + value1_formatted=$(printf "%.4f" "$value1") + value2_formatted=$(printf "%.4f" "$value2") + + printf "%-25s %-27s %-27s %-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$diff" +done +printf "Namespace Memory (MB)\n" +for namespace in "${ALL_MEM_NAMESPACES[@]}"; do + value1="${MEM1_VALUES[$namespace]}" + value2="${MEM2_VALUES[$namespace]}" + + if [ -z "$value1" ]; then value1=0; fi + if [ -z "$value2" ]; then value2=0; fi + diff=$(echo "$value2 - $value1" | bc -l) + sign="" + if (( $(echo "$diff < 0" | bc -l) )); then + sign="-" + diff=$(echo "$value1 - $value2" | bc -l) + else + sign="+" + fi + diff=$(echo "scale=2; $diff / 1048576" | bc) + + value1_formatted=$(echo "scale=2; $value1 / 1048576" | bc) + value2_formatted=$(echo "scale=2; $value2 / 1048576" | bc) + + printf "%-25s %-27s %-27s %s%-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$sign" "$diff" +done +printf "Controller 99th Reconcile (s)\n" +for namespace in "${ALL_RECONCILE_CONTROLLERS[@]}"; do + value1="${RECONCILE1_VALUES[$namespace]}" + value2="${RECONCILE2_VALUES[$namespace]}" + + if [ -z "$value1" ]; then value1=0; fi + if [ -z "$value2" ]; then value2=0; fi + diff=$(compare_values "$value1" "$value2" "%.4f") + + value1_formatted=$(printf "%.4f" "$value1") + value2_formatted=$(printf "%.4f" "$value2") + + printf "%-25s %-27s %-27s %-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$diff" +done diff --git a/scale_test/readme.md b/scale_test/readme.md index d9c7d2a0..e5ad48d8 100644 --- a/scale_test/readme.md +++ b/scale_test/readme.md @@ -179,3 +179,62 @@ Some common transforms are: * "Sort by" Inspecting some of the existing panels in the Kuadrant dashboard and kube-burner dashboards at https://github.com/kube-burner/kube-burner/tree/main/examples/grafana-dashboards can give inspiration as well. + +## Performance Comparison Script + +The `./compare.sh` script allows you to compare key performance metrics between two test runs. It fetches metrics from an Elasticsearch instance and generates a report highlighting the differences in performance between the two specified runs, similar to the comparison Grafana dashboard. Values are calculated based on the average of most recent values. + +You can specify the UUIDs of the two test runs you wish to compare by setting the `UUID1` and `UUID2` environment variables when running the script. + +```bash +UUID1= UUID2= ./compare.sh +``` + +**Example**: + +```bash +UUID1=926c7847-95bd-4ed2-bf80-2cad6746db9c UUID2=c1195a3d-6ceb-4468-951c-d0492ef7c79c ./compare.sh +``` + +### Automatic UUID Selection + +If you do not specify `UUID1` and `UUID2`, the script will automatically fetch the two most recent test runs from Elasticsearch and compare them. + +```bash +./compare.sh +``` + +## Example Output + +``` +Metric UUID 1 (926c7847) UUID 2 (c1195a3d) Diff +Timestamp 2024-10-18T12:01:11.588843Z 2024-10-18T12:50:47.638731Z +Elapsed Time 152 220 +68 + +Namespace CPU (Average Values) +Namespace 926c7847 c1195a3d Diff +kuadrant-system 0.0119 0.0122 +0.0003 +scale-test-0 0.0006 0.1076 +0.1070 +gateway-system 0.0022 0.0019 -0.0003 +istio-system 0.0071 0.0317 +0.0246 + +Namespace Memory (MB) +Namespace 926c7847 c1195a3d Diff +kuadrant-system 181.78 152.12 -29.65 +scale-test-0 168.64 267.74 +99.09 +gateway-system 40.30 39.59 -0.70 +istio-system 114.27 119.07 +4.79 + +Controller 99th Reconcile (s) +Controller 926c7847 c1195a3d Diff +dnsrecord 5.8800 2.9600 -2.9200 +authpolicy 0.1835 0.0955 -0.0880 +dnspolicy 0.1410 0.1423 +0.0012 +kuadrant 0.0980 0.1475 +0.0495 +limitador 0.0980 0.3975 +0.2995 +gateway 0.1474 0.0885 -0.0588 +controller 3.4350 1.2150 -2.2200 +tlspolicy 3.8300 0.0959 -3.7341 +ratelimitpolicy 0.1760 0.0902 -0.0858 +authorino 0.0000 0.6955 +0.6955 +```