-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into analyze_port
- Loading branch information
Showing
13 changed files
with
315 additions
and
93 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,50 +1,81 @@ | ||
| name: "Run benchmark" | ||
|
|
||
| on: | ||
| pull_request_target: | ||
| types: [assigned, opened, synchronize, reopened, edited] | ||
| push: | ||
| branches: | ||
| - main | ||
|
|
||
| permissions: | ||
| contents: write | ||
| pull-requests: write | ||
| issues: write | ||
|
|
||
| jobs: | ||
| build: | ||
| runs-on: benchmarking-runner | ||
| if: github.event.head_commit.message != 'Update performance results in README.md' | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| strategy: | ||
| matrix: | ||
| service: [apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit] | ||
| steps: | ||
| - name: Checkout (GitHub) | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | ||
|
|
||
| - name: Build devcontainer and run benchmarks | ||
| - name: Build devcontainer and run setup and benchmark | ||
| uses: devcontainers/[email protected] | ||
| with: | ||
| imageName: graphql-benchmarks | ||
| push: never | ||
| runCmd: | | ||
| bash ./setup.sh | ||
| bash ./run_benchmarks.sh | ||
| bash ./graphql/${{ matrix.service }}/setup.sh | ||
| bash run_benchmarks.sh ${{ matrix.service }} | ||
| - name: List benchmark files | ||
| run: | | ||
| ls -la bench*.txt || echo "No matching files found" | ||
| - name: Upload benchmark results | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: benchmark-results | ||
| path: bench*.txt | ||
|
|
||
|
|
||
| analyze: | ||
| needs: benchmark | ||
| runs-on: benchmarking-runner | ||
| steps: | ||
| - name: Checkout (GitHub) | ||
| uses: actions/checkout@v4 | ||
|
|
||
| - name: Download all benchmark results | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: benchmark-results | ||
| path: . | ||
|
|
||
| - name: List downloaded artifacts | ||
| run: ls -la bench*.txt || echo "No matching files found" | ||
|
|
||
| - name: Analyze results | ||
| run: | | ||
| bash run_analyze_script.sh | ||
| - name: Print benchmark results | ||
| run: cat ./results.md | ||
|
|
||
| - name: Comment benchmark results on PR | ||
| if: github.event_name == 'pull_request_target' | ||
| uses: peter-evans/commit-comment@v3 | ||
| with: | ||
| sha: ${{ github.event.pull_request.head.sha }} | ||
| body-path: "results.md" | ||
| reactions: eyes | ||
|
|
||
| - name: Commit and push changes (on main branch) | ||
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | ||
| uses: stefanzweifel/git-auto-commit-action@v5 | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,167 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Install gnuplot | ||
| sudo apt-get update && sudo apt-get install -y gnuplot | ||
|
|
||
| function extractMetric() { | ||
| local file="$1" | ||
| local metric="$2" | ||
| grep "$metric" "$file" | awk '{print $2}' | sed 's/ms//' | ||
| } | ||
|
|
||
| function average() { | ||
| echo "$@" | awk '{for(i=1;i<=NF;i++) s+=$i; print s/NF}' | ||
| } | ||
|
|
||
| declare -A formattedServerNames | ||
| formattedServerNames=( | ||
| ["tailcall"]="Tailcall" | ||
| ["gqlgen"]="Gqlgen" | ||
| ["apollo"]="Apollo GraphQL" | ||
| ["netflixdgs"]="Netflix DGS" | ||
| ["caliban"]="Caliban" | ||
| ["async_graphql"]="async-graphql" | ||
| ["hasura"]="Hasura" | ||
| ["graphql_jit"]="GraphQL JIT" | ||
| ) | ||
|
|
||
| servers=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") | ||
| resultFiles=("$@") | ||
| declare -A avgReqSecs | ||
| declare -A avgLatencies | ||
|
|
||
| # Extract metrics and calculate averages | ||
| for idx in "${!servers[@]}"; do | ||
| startIdx=$((idx * 3)) | ||
| reqSecVals=() | ||
| latencyVals=() | ||
| for j in 0 1 2; do | ||
| fileIdx=$((startIdx + j)) | ||
| reqSecVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Requests/sec")) | ||
| latencyVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Latency")) | ||
| done | ||
| avgReqSecs[${servers[$idx]}]=$(average "${reqSecVals[@]}") | ||
| avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}") | ||
| done | ||
|
|
||
| # Generating data files for gnuplot | ||
| reqSecData="/tmp/reqSec.dat" | ||
| latencyData="/tmp/latency.dat" | ||
|
|
||
| echo "Server Value" >"$reqSecData" | ||
| for server in "${servers[@]}"; do | ||
| echo "$server ${avgReqSecs[$server]}" >>"$reqSecData" | ||
| done | ||
|
|
||
| echo "Server Value" >"$latencyData" | ||
| for server in "${servers[@]}"; do | ||
| echo "$server ${avgLatencies[$server]}" >>"$latencyData" | ||
| done | ||
|
|
||
| whichBench=1 | ||
| if [[ $1 == bench2* ]]; then | ||
| whichBench=2 | ||
| elif [[ $1 == bench3* ]]; then | ||
| whichBench=3 | ||
| fi | ||
|
|
||
| reqSecHistogramFile="req_sec_histogram${whichBench}.png" | ||
| latencyHistogramFile="latency_histogram${whichBench}.png" | ||
|
|
||
| # Plotting using gnuplot | ||
| gnuplot <<-EOF | ||
| set term pngcairo size 1280,720 enhanced font "Courier,12" | ||
| set output "$reqSecHistogramFile" | ||
| set style data histograms | ||
| set style histogram cluster gap 1 | ||
| set style fill solid border -1 | ||
| set xtics rotate by -45 | ||
| set boxwidth 0.9 | ||
| set title "Requests/Sec" | ||
| stats "$reqSecData" using 2 nooutput | ||
| set yrange [0:STATS_max*1.2] | ||
| set key outside right top | ||
| plot "$reqSecData" using 2:xtic(1) title "Req/Sec" | ||
| set output "$latencyHistogramFile" | ||
| set title "Latency (in ms)" | ||
| stats "$latencyData" using 2 nooutput | ||
| set yrange [0:STATS_max*1.2] | ||
| plot "$latencyData" using 2:xtic(1) title "Latency" | ||
| EOF | ||
|
|
||
| # Move PNGs to assets | ||
| mkdir -p assets | ||
| mv $reqSecHistogramFile assets/ | ||
| mv $latencyHistogramFile assets/ | ||
|
|
||
| # Declare an associative array for server RPS | ||
| declare -A serverRPS | ||
|
|
||
| # Populate the serverRPS array | ||
| for server in "${servers[@]}"; do | ||
| serverRPS[$server]=${avgReqSecs[$server]} | ||
| done | ||
|
|
||
| # Get the servers sorted by RPS in descending order | ||
| IFS=$'\n' sortedServers=($(for server in "${!serverRPS[@]}"; do echo "$server ${serverRPS[$server]}"; done | sort -rn -k2 | cut -d' ' -f1)) | ||
|
|
||
| echo "Sorted servers: ${sortedServers[@]}" | ||
| lastServer="${sortedServers[-1]}" | ||
| lastServerReqSecs=${avgReqSecs[$lastServer]} | ||
|
|
||
| # Start building the resultsTable | ||
| if [[ $whichBench == 1 ]]; then | ||
| resultsTable="<!-- PERFORMANCE_RESULTS_START -->\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| $whichBench | \`{ posts { id userId title user { id name email }}}\` |" | ||
| elif [[ $whichBench == 2 ]]; then | ||
| resultsTable="| $whichBench | \`{ posts { title }}\` |" | ||
| elif [[ $whichBench == 3 ]]; then | ||
| resultsTable="| $whichBench | \`{ greet }\` |" | ||
| fi | ||
|
|
||
| # Build the resultsTable with sorted servers and formatted numbers | ||
| for server in "${sortedServers[@]}"; do | ||
| formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') | ||
| formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') | ||
| # Calculate the relative performance | ||
| relativePerformance=$(echo "${avgReqSecs[$server]} $lastServerReqSecs" | awk '{printf "%.2f", $1 / $2}') | ||
|
|
||
| resultsTable+="\n|| [${formattedServerNames[$server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |" | ||
| done | ||
|
|
||
| if [[ $whichBench == 3 ]]; then | ||
| resultsTable+="\n\n<!-- PERFORMANCE_RESULTS_END -->" | ||
| fi | ||
|
|
||
| echo "resultsTable: $resultsTable" | ||
|
|
||
| # Print the results table in a new file | ||
| resultsFile="results.md" | ||
| echo -e $resultsTable >> $resultsFile | ||
|
|
||
|
|
||
| if [[ $whichBench == 3 ]]; then | ||
| finalResults=$(printf '%s\n' "$(cat $resultsFile)" | sed 's/$/\\n/'| tr -d '\n') | ||
| # Remove the last newline character | ||
| finalResults=${finalResults::-2} | ||
|
|
||
| # Print the results as a table in the terminal | ||
| echo -e $finalResults | sed "s/<!-- PERFORMANCE_RESULTS_START-->//;s/<!-- PERFORMANCE_RESULTS_END-->//" | ||
| # Check if the markers are present | ||
| if grep -q "PERFORMANCE_RESULTS_START" README.md; then | ||
| # Replace the old results with the new results | ||
| sed -i "/PERFORMANCE_RESULTS_START/,/PERFORMANCE_RESULTS_END/c\\$finalResults" README.md | ||
| else | ||
| # Append the results at the end of the README.md file | ||
| echo -e "\n$finalResults" >> README.md | ||
| fi | ||
| fi | ||
|
|
||
| # Move the generated images to the assets folder | ||
| mv $reqSecHistogramFile assets/ | ||
| mv $latencyHistogramFile assets/ | ||
|
|
||
| # Delete the result TXT files | ||
| for file in "${resultFiles[@]}"; do | ||
| rm "$file" | ||
| done |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Oops, something went wrong.
9cccb7cThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
{ posts { id userId title user { id name email }}}29,936.403.33211.26x2,016.1849.9814.23x1,712.3558.3612.08x1,364.9172.939.63x809.03122.675.71x375.06163.392.65x268.25366.501.89x141.71506.801.00x{ posts { title }}59,464.001.6768.27x9,741.4410.3811.18x9,572.2810.8110.99x2,227.6546.402.56x1,762.8556.662.02x1,619.0469.121.86x1,405.4571.061.61x871.07114.541.00x{ greet }67,270.101.0725.15x60,461.501.6622.61x48,630.802.0918.18x47,768.605.0517.86x8,345.8914.393.12x8,052.0612.733.01x5,307.8118.821.98x2,674.5337.391.00x