From db01af60dce77185e7aeaf84977ed720bccda6c4 Mon Sep 17 00:00:00 2001 From: Panagiotis Date: Tue, 10 Sep 2024 15:18:01 +0300 Subject: [PATCH] Improve results and break CI on errors --- .github/workflows/benchmark-pull-request.yaml | 9 ++ check.sh | 33 +++++++ printjson.sh | 48 ++++++++++ tojson.sh | 96 +++++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100755 check.sh create mode 100755 printjson.sh create mode 100755 tojson.sh diff --git a/.github/workflows/benchmark-pull-request.yaml b/.github/workflows/benchmark-pull-request.yaml index 64c8c8a..ecdc0d8 100644 --- a/.github/workflows/benchmark-pull-request.yaml +++ b/.github/workflows/benchmark-pull-request.yaml @@ -37,3 +37,12 @@ jobs: - name: Run Benchmarks run: cargo run + + - name: Prepare JSON + run: ./tojson.sh + + - name: Print Results + run: ./printjson.sh + + - name: Check Result + run: ./check.sh diff --git a/check.sh b/check.sh new file mode 100755 index 0000000..e4b6f1b --- /dev/null +++ b/check.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Loop through subfolders +for dir in ./results/*/; do + if [ -d "$dir" ]; then + echo "Checking $dir" + + # Construct full path to bench_1.json + json_file="$dir/bench_1.json" + + # Check if file exists + if [ ! -f "$json_file" ]; then + echo "Error: $json_file not found" + continue + fi + + # Read the JSON file using jq + socket_errors=$(jq -r '.socket_errors | {connect: .connect, read: .read, write: .write, timeout: .timeout}' "$json_file") + + + # Check if any socket error value is greater than zero + if [[ $socket_errors =~ ([0-9]+) ]]; then + connect=$(echo $socket_errors | grep -oP '"connect":\s*\K[0-9]+' | head -n 1 || echo 0) + read=$(echo $socket_errors | grep -oP '"read":\s*\K[0-9]+' | head -n 1 || echo 0) + write=$(echo $socket_errors | grep -oP '"write":\s*\K[0-9]+' | head -n 1 || echo 0) + timeout=$(echo $socket_errors | grep -oP '"timeout":\s*\K[0-9]+' | head -n 1 || echo 0) + if [[ $connect > 0 || $read > 0 || $write > 0 || $timeout > 0 ]]; then + echo "Error: Socket errors detected in $dir" + exit 1 + fi + fi + fi +done diff --git a/printjson.sh b/printjson.sh new file mode 100755 index 0000000..51eb9bb --- /dev/null +++ b/printjson.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +function format_json_table() { + local json_file="$1" + + # Parse the JSON and flatten it + local output=$(jq -r ' + ["Category", "Average", "Standard Deviation", "Max", "Stdev Percent"], + ["Latency", .latency.avg, .latency.stdev, .latency.max, .latency.stdev_percent], + ["Requests", .requests.avg, .requests.stdev, .requests.max, .requests.stdev_percent], + ["Total Requests", .total_requests, "", "", ""], + ["Memory Usage", .memory_usage, "", "", ""], + ["Requests Per Second", .requests_per_second, "", "", ""], + ["Transfer Per Second", .transfer_per_second, "", "", ""], + ["Socket Errors (Connect)", .socket_errors.connect, "", "", ""], + ["Socket Errors (Read)", .socket_errors.read, "", "", ""], + ["Socket Errors (Write)", .socket_errors.write, "", "", ""], + ["Socket Errors (Timeout)", .socket_errors.timeout, "", "", ""] + | @tsv' "$json_file") + + # Print the header for the markdown table + echo "| Category | Average | Standard Deviation | Max | Stdev Percent |" + echo "|-----------------------|-----------|---------------------|------------|---------------|" + + # Use while loop to print each line from the jq output into markdown table format + while IFS=$'\t' read -r category avg stdev max stdev_percent; do + printf "| %-22s | %-9s | %-19s | %-10s | %-13s |\n" "$category" "$avg" "$stdev" "$max" "$stdev_percent" + done <<< "$output" +} + +# Usage +# Iterate through directories in /results +for dir in ./results/*/; do + if [ -d "$dir" ]; then + echo "Results for $dir" + + # Construct full path to bench_1.json + json_file="$dir/bench_1.json" + + # Check if file exists + if [ ! -f "$json_file" ]; then + echo "Error: $json_file not found" + continue + fi + + format_json_table $json_file + fi +done diff --git a/tojson.sh b/tojson.sh new file mode 100755 index 0000000..974c295 --- /dev/null +++ b/tojson.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Function to parse bench_1.out file +parse_bench_file() { + local file_path="$1" + + # Parse latency statistics + latency_avg=$(awk '/Latency/{print $2}' "$file_path") + latency_stdev=$(awk '/Latency/{print $3}' "$file_path") + latency_max=$(awk '/Latency/{print $4}' "$file_path") + latency_stdev_percent=$(awk '/Latency/{print $5}' "$file_path") + + # Parse requests statistics + req_per_sec_avg=$(awk '/Req\/Sec/{print $2}' "$file_path") + req_per_sec_stdev=$(awk '/Req\/Sec/{print $3}' "$file_path") + req_per_sec_max=$(awk '/Req\/Sec/{print $4}' "$file_path") + req_per_sec_stdev_percent=$(awk '/Req\/Sec/{print $5}' "$file_path") + + # Parse total requests and memory usage + total_requests=$(awk '/requests in/{print $1}' "$file_path") + memory_usage=$(awk '/requests in/{print $5}' "$file_path") + + # Parse Requests/sec metric + requests_sec=$(grep -i "Requests/sec" "$file_path" | awk '{print $NF}') + + # Parse Transfer/sec metric + transfer_sec=$(grep -i "Transfer/sec" "$file_path" | awk '{print $NF}') + + # Check if Socket errors line exists + if grep -q "Socket errors:" "$file_path"; then + # Parse socket errors + connect_errors=$(grep "Socket errors:" "$file_path" | cut -d',' -f1 | awk '{print $NF}') + read_errors=$(grep "Socket errors:" "$file_path" | cut -d',' -f2 | awk '{print $NF}') + write_errors=$(grep "Socket errors:" "$file_path" | cut -d',' -f3 | awk '{print $NF}') + timeout_errors=$(grep "Socket errors:" "$file_path" | cut -d',' -f4 | awk '{print $NF}') + + socket_errors_json="\"socket_errors\": { + \"connect\": $connect_errors, + \"read\": $read_errors, + \"write\": $write_errors, + \"timeout\": $timeout_errors + }" + else + # Default values if Socket errors line is not present + socket_errors_json="\"socket_errors\": { + \"connect\": 0, + \"read\": 0, + \"write\": 0, + \"timeout\": 0 + }" + fi + + echo "{ + \"latency\": { + \"avg\": \"$latency_avg\", + \"stdev\": \"$latency_stdev\", + \"max\": \"$latency_max\", + \"stdev_percent\": \"$latency_stdev_percent\" + }, + \"requests\": { + \"avg\": \"$req_per_sec_avg\", + \"stdev\": \"$req_per_sec_stdev\", + \"max\": \"$req_per_sec_max\", + \"stdev_percent\": \"$req_per_sec_stdev_percent\" + }, + \"total_requests\": \"$total_requests\", + \"memory_usage\": \"$memory_usage\", + \"requests_per_second\": \"$requests_sec\", + \"transfer_per_second\": \"$transfer_sec\", + $socket_errors_json + }" +} + +# Iterate through directories in /results +for dir in ./results/*/; do + if [ -d "$dir" ]; then + bench_file="$dir/bench_1.out" + + if [ -f "$bench_file" ]; then + echo "Processing $bench_file:" + + # Get the directory path of bench_1.out + output_dir=$(dirname "$bench_file") + + # Create the JSON file name + json_file="$output_dir/bench_1.json" + + # Parse the file and save output to bench_1.json + parse_bench_file "$bench_file" > "$json_file" + + echo "JSON saved to $json_file" + else + echo "Warning: $bench_file not found." + fi + fi +done \ No newline at end of file