refactor: improve results and break CI on errors

.github/workflows/benchmark-pull-request.yaml

@@ -52,3 +52,12 @@ jobs:
 
       - name: Run Benchmarks
         run: cargo run -- --project ${{ github.event.pull_request.head.repo.owner.login }}
+
+      - name: Prepare JSON
+        run: ./tojson.sh
+
+      - name: Print Results
+        run: ./printjson.sh
+
+      - name: Check Result
+        run: ./check.sh

check.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Loop through subfolders
+for dir in ./results/*/; do
+    if [ -d "$dir" ]; then
+        echo "Checking $dir"
+
+        # Construct full path to bench_1.json
+        json_file="$dir/bench_1.json"
+
+        # Check if file exists
+        if [ ! -f "$json_file" ]; then
+            echo "Error: $json_file not found"
+            continue
+        fi
+
+        # Read the JSON file using jq
+        socket_errors=$(jq -r '.socket_errors | {connect: .connect, read: .read, write: .write, timeout: .timeout}' "$json_file")
+
+
+        # Check if any socket error value is greater than zero
+        if [[ $socket_errors =~ ([0-9]+) ]]; then
+            connect=$(echo $socket_errors | grep -oP '"connect":\s*\K[0-9]+' | head -n 1 || echo 0)
+            read=$(echo $socket_errors | grep -oP '"read":\s*\K[0-9]+' | head -n 1 || echo 0)
+            write=$(echo $socket_errors | grep -oP '"write":\s*\K[0-9]+' | head -n 1 || echo 0)
+            timeout=$(echo $socket_errors | grep -oP '"timeout":\s*\K[0-9]+' | head -n 1 || echo 0)
+            if [[ $connect > 0 || $read > 0 || $write > 0 || $timeout > 0 ]]; then
+                echo "Error: Socket errors detected in $dir"
+                exit 1
+            fi
+        fi
+    fi
+done

printjson.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+function format_json_table() {
+    local json_file="$1"
+
+    # Parse the JSON and flatten it
+    local output=$(jq -r '
+        ["Category", "Average", "Standard Deviation", "Max", "Stdev Percent"],
+        ["Latency", .latency.avg, .latency.stdev, .latency.max, .latency.stdev_percent],
+        ["Requests", .requests.avg, .requests.stdev, .requests.max, .requests.stdev_percent],
+        ["Total Requests", .total_requests, "", "", ""],
+        ["Memory Usage", .memory_usage, "", "", ""],
+        ["Requests Per Second", .requests_per_second, "", "", ""],
+        ["Transfer Per Second", .transfer_per_second, "", "", ""],
+        ["Socket Errors (Connect)", .socket_errors.connect, "", "", ""],
+        ["Socket Errors (Read)", .socket_errors.read, "", "", ""],
+        ["Socket Errors (Write)", .socket_errors.write, "", "", ""],
+        ["Socket Errors (Timeout)", .socket_errors.timeout, "", "", ""]
+        | @tsv' "$json_file")
+
+    # Print the header for the markdown table
+    echo "| Category              | Average   | Standard Deviation  | Max        | Stdev Percent |"
+    echo "|-----------------------|-----------|---------------------|------------|---------------|"
+
+    # Use while loop to print each line from the jq output into markdown table format
+    while IFS=$'\t' read -r category avg stdev max stdev_percent; do
+        printf "| %-22s | %-9s | %-19s | %-10s | %-13s |\n" "$category" "$avg" "$stdev" "$max" "$stdev_percent"
+    done <<< "$output"
+}
+
+# Usage
+# Iterate through directories in /results
+for dir in ./results/*/; do
+    if [ -d "$dir" ]; then
+        echo "Results for $dir"
+
+        # Construct full path to bench_1.json
+        json_file="$dir/bench_1.json"
+
+        # Check if file exists
+        if [ ! -f "$json_file" ]; then
+            echo "Error: $json_file not found"
+            continue
+        fi
+
+        format_json_table $json_file
+    fi
+done

tojson.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+# Function to parse bench_1.out file
+parse_bench_file() {
+    local file_path="$1"
+
+    # Parse latency statistics
+    latency_avg=$(awk '/Latency/{print $2}' "$file_path")
+    latency_stdev=$(awk '/Latency/{print $3}' "$file_path")
+    latency_max=$(awk '/Latency/{print $4}' "$file_path")
+    latency_stdev_percent=$(awk '/Latency/{print $5}' "$file_path")
+
+    # Parse requests statistics
+    req_per_sec_avg=$(awk '/Req\/Sec/{print $2}' "$file_path")
+    req_per_sec_stdev=$(awk '/Req\/Sec/{print $3}' "$file_path")
+    req_per_sec_max=$(awk '/Req\/Sec/{print $4}' "$file_path")
+    req_per_sec_stdev_percent=$(awk '/Req\/Sec/{print $5}' "$file_path")
+
+    # Parse total requests and memory usage
+    total_requests=$(awk '/requests in/{print $1}' "$file_path")
+    memory_usage=$(awk '/requests in/{print $5}' "$file_path")
+
+    # Parse Requests/sec metric
+    requests_sec=$(grep -i "Requests/sec" "$file_path" | awk '{print $NF}')
+
+    # Parse Transfer/sec metric
+    transfer_sec=$(grep -i "Transfer/sec" "$file_path" | awk '{print $NF}')
+
+    # Check if Socket errors line exists
+    if grep -q "Socket errors:" "$file_path"; then
+        # Parse socket errors
+        connect_errors=$(grep "Socket errors:" "$file_path" | cut -d',' -f1 | awk '{print $NF}')
+        read_errors=$(grep "Socket errors:" "$file_path" | cut -d',' -f2 | awk '{print $NF}')
+        write_errors=$(grep "Socket errors:" "$file_path" | cut -d',' -f3 | awk '{print $NF}')
+        timeout_errors=$(grep "Socket errors:" "$file_path" | cut -d',' -f4 | awk '{print $NF}')
+
+        socket_errors_json="\"socket_errors\": {
+            \"connect\": $connect_errors,
+            \"read\": $read_errors,
+            \"write\": $write_errors,
+            \"timeout\": $timeout_errors
+        }"
+    else
+        # Default values if Socket errors line is not present
+        socket_errors_json="\"socket_errors\": {
+            \"connect\": 0,
+            \"read\": 0,
+            \"write\": 0,
+            \"timeout\": 0
+        }"
+    fi
+
+    echo "{
+        \"latency\": {
+            \"avg\": \"$latency_avg\",
+            \"stdev\": \"$latency_stdev\",
+            \"max\": \"$latency_max\",
+            \"stdev_percent\": \"$latency_stdev_percent\"
+        },
+        \"requests\": {
+            \"avg\": \"$req_per_sec_avg\",
+            \"stdev\": \"$req_per_sec_stdev\",
+            \"max\": \"$req_per_sec_max\",
+            \"stdev_percent\": \"$req_per_sec_stdev_percent\"
+        },
+        \"total_requests\": \"$total_requests\",
+        \"memory_usage\": \"$memory_usage\",
+        \"requests_per_second\": \"$requests_sec\",
+        \"transfer_per_second\": \"$transfer_sec\",
+        $socket_errors_json
+    }"
+}
+
+# Iterate through directories in /results
+for dir in ./results/*/; do
+    if [ -d "$dir" ]; then
+        bench_file="$dir/bench_1.out"
+
+        if [ -f "$bench_file" ]; then
+            echo "Processing $bench_file:"
+
+            # Get the directory path of bench_1.out
+            output_dir=$(dirname "$bench_file")
+
+            # Create the JSON file name
+            json_file="$output_dir/bench_1.json"
+
+            # Parse the file and save output to bench_1.json
+            parse_bench_file "$bench_file" > "$json_file"
+
+            echo "JSON saved to $json_file"
+        else
+            echo "Warning: $bench_file not found."
+        fi
+    fi
+done