ci: Update benchmark.yml #184
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| push: | |
| paths-ignore: ["docs/**", "**.md"] | |
| branches: | |
| - main | |
| pull_request: | |
| paths-ignore: ["docs/**", "**.md"] | |
| types: [opened, reopened, synchronize, labeled] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| macro_benchmark: | |
| name: Macro Benchmarks | |
| if: "contains(github.event.pull_request.labels.*.name, 'ci: benchmark') || github.event_name == 'push'" | |
| runs-on: benchmarking-runner | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Stable Toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - name: Run Cargo Build | |
| run: cargo build --release | |
| - name: Run Tailcall | |
| run: | | |
| TAILCALL_LOG_LEVEL=error ./target/release/tailcall start ci-benchmark/benchmark.graphql & | |
| - name: Install Wrk | |
| run: | | |
| sudo apt-get install -y wrk | |
| - name: Run Test Query | |
| run: | | |
| curl -i -X POST -d '{"query": "{posts{title}}"}' http://localhost:8000/graphql -H "Content-Type: application/json" | |
| - name: Warmup Wrk | |
| working-directory: ci-benchmark | |
| run: | | |
| wrk -d 10 -t 4 -c 100 -s wrk.lua http://localhost:8000/graphql | |
| - id: run_wrk | |
| name: Run Wrk | |
| working-directory: ci-benchmark | |
| run: | | |
| wrk -d 30 -t 4 -c 100 -s wrk.lua http://localhost:8000/graphql > wrk-output.txt | |
| - id: convert_wrk_output_markdown | |
| name: Convert Output to Markdown | |
| working-directory: ci-benchmark | |
| run: | | |
| node wrk-output-to-md.js wrk-output.txt > body.md | |
| - id: cat_md | |
| name: Cat Markdown | |
| working-directory: ci-benchmark | |
| run: | | |
| cat body.md | |
| - name: "Upload Artifact" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: body | |
| path: ci-benchmark/body.md | |
| Track_and_Cache_Benchmarks: | |
| name: Track and Cache Micro Benchmarks | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Install Bencher CLI | |
| uses: bencherdev/bencher@main | |
| - name: Run and track Benchmarks | |
| run: | | |
| echo "hello ${{ secrets.HELLO }}" | |
| bencher run \ | |
| --project tailcall-2192953494 \ | |
| --branch main \ | |
| --testbed benchmarking-runner-12 \ | |
| --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxODEwNDE3NzU0LCJpYXQiOjE3MTA0MTc3NTQsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.ET5NU6zbaBVyBcp5cAz0VzO_wSLKFTIueuYzebfMOJk \ | |
| --adapter rust_criterion \ | |
| --file benches/base.txt | |
| - name: Cache Criterion Benchmarks Json | |
| uses: actions/cache@v4 | |
| with: | |
| path: benches/base.txt | |
| key: criterion_benchmarks_${{ github.sha }} | |
| Criterion_Compare: | |
| name: Comparing Micro Benchmarks | |
| if: "contains(github.event.pull_request.labels.*.name, 'ci: benchmark')" | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - name: Run Criterion Benchmarks | |
| run: | | |
| cargo bench > benches/change.txt | |
| - name: Restore file | |
| uses: actions/cache@v4 | |
| with: | |
| path: benches/base.txt | |
| key: criterion_benchmarks_${{ github.event.pull_request.base.sha }} | |
| fail-on-cache-miss: true | |
| - name: Install Bencher CLI | |
| uses: bencherdev/bencher@main | |
| - name: Create PR threshold | |
| run: | | |
| bencher threshold create \ | |
| --project tailcall-2192953494 \ | |
| --branch '${{ github.head_ref }}' \ | |
| --testbed benchmarking-runner-12 \ | |
| --adapter rust_criterion \ | |
| --measure latency \ | |
| --test t \ | |
| --upper-boundary 0.90 \ | |
| --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxODEwNDE3NzU0LCJpYXQiOjE3MTA0MTc3NTQsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.ET5NU6zbaBVyBcp5cAz0VzO_wSLKFTIueuYzebfMOJk \ | |
| - name: Track PR Benchmarks | |
| run: | | |
| bencher run \ | |
| --branch '${{ github.head_ref }}' \ | |
| --token "${{ secrets.BENCHER_API_TOKEN }}" \ | |
| --ci-number '${{ github.event.pull_request.number }}' \ | |
| --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxODEwNDE3NzU0LCJpYXQiOjE3MTA0MTc3NTQsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.ET5NU6zbaBVyBcp5cAz0VzO_wSLKFTIueuYzebfMOJk \ | |
| --err \ | |
| --file benches/change.txt |