diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 61a20f11e3..2aba2d2397 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -69,13 +69,13 @@ jobs: name: body path: ci-benchmark/body.md - Track_and_Cache_Benchmarks: - name: Track and Cache Micro Benchmarks + Cache_Benchmarks: + name: Cache Micro Benchmarks result if: (github.event_name == 'push' && github.ref == 'refs/heads/main') permissions: pull-requests: write contents: write - runs-on: ubuntu-latest + runs-on: benchmarking-runner steps: - name: Check out code uses: actions/checkout@v4 @@ -83,30 +83,23 @@ jobs: - name: Install Rust uses: actions-rust-lang/setup-rust-toolchain@v1 - - name: Install Bencher CLI - uses: bencherdev/bencher@main - - - name: Run and track Benchmarks + - name: Run Benchmarks run: | - cargo bench > benches/base.txt - bencher run \ - --project tailcall-2192953494 \ - --branch main \ - --testbed benchmarking-runner \ - --token "${{ secrets.BENCHER_API_TOKEN }}" \ - --adapter rust_criterion \ - --file benches/base.txt + cargo install cargo-criterion rust-script + cargo criterion --message-format=json > benches/main_benchmarks.json + ./scripts/json_to_md.rs benches/main_benchmarks.json > benches/main_benchmarks.md + cat benches/main_benchmarks.md - name: Cache Criterion Benchmarks Json uses: actions/cache@v4 with: - path: benches/base.txt + path: benches/main_benchmarks.json key: criterion_benchmarks_${{ github.sha }} Criterion_Compare: name: Comparing Micro Benchmarks if: "contains(github.event.pull_request.labels.*.name, 'ci: benchmark')" - runs-on: ubuntu-latest + runs-on: benchmarking-runner permissions: pull-requests: write contents: write @@ -121,36 +114,22 @@ jobs: - name: Run Criterion Benchmarks run: | - cargo bench > benches/change.txt + cargo install cargo-criterion rust-script + cargo criterion --message-format=json > benches/benchmarks.json + ./scripts/json_to_md.rs benches/benchmarks.json > benches/change_benchmarks.md + + - name: Print Criterion Benchmarks + run: cat benches/change_benchmarks.md - name: Restore file uses: actions/cache@v4 with: - path: benches/base.txt + path: benches/main_benchmarks.json key: criterion_benchmarks_${{ github.event.pull_request.base.sha }} fail-on-cache-miss: true - - name: Install Bencher CLI - uses: bencherdev/bencher@main + - name: Print Benchmark Comparision + run: ./scripts/criterion_compare.rs benches/main_benchmarks.json benches/benchmarks.json table - - name: Create PR threshold - run: | - bencher threshold create \ - --project tailcall-2192953494 \ - --branch '${{ github.head_ref }}' \ - --testbed benchmarking-runner \ - --adapter rust_criterion \ - --measure latency \ - --test t \ - --upper-boundary 0.90 \ - --token "${{ secrets.BENCHER_API_TOKEN }}" - - - name: Track PR Benchmarks - run: | - bencher run \ - --branch '${{ github.head_ref }}' \ - --token "${{ secrets.BENCHER_API_TOKEN }}" \ - --ci-number '${{ github.event.pull_request.number }}' \ - --github-actions "${{ secrets.GITHUB_TOKEN }}" \ - --err \ - --file benches/change.txt + - name: Check Degradation + run: ./scripts/criterion_compare.rs benches/main_benchmarks.json benches/benchmarks.json check