ci: test pr

.github/workflows/benchmark.yml

@@ -69,67 +69,98 @@ jobs:
           name: body
           path: ci-benchmark/body.md
 
-  Cache_Benchmarks:
-    name: Cache Micro Benchmarks result
+  Track_and_Cache_Benchmarks:
+    name: Track and Cache Micro Benchmarks
     if: (github.event_name == 'push' && github.ref == 'refs/heads/main')
     permissions:
       pull-requests: write
       contents: write
-    runs-on: benchmarking-runner
+    runs-on: ubuntu-latest
     steps:
       - name: Check out code
         uses: actions/checkout@v4
 
       - name: Install Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
 
-      - name: Run Benchmarks
+      - name: Install Bencher CLI
+        uses: bencherdev/bencher@main
+
+      - name: Run and track Benchmarks
         run: |
-          cargo install cargo-criterion rust-script
-          cargo criterion --message-format=json > benches/main_benchmarks.json
-          ./scripts/json_to_md.rs benches/main_benchmarks.json > benches/main_benchmarks.md
-          cat benches/main_benchmarks.md
+          cargo bench > benches/base.txt
+          bencher run \
+          --project tailcall-2192953494 \
+          --branch main \
+          --testbed benchmarking-runner-12 \
+          --token "${{ secrets.BENCHER_API_TOKEN }}" \
+          --adapter rust_criterion \
+          --file benches/base.txt
 
       - name: Cache Criterion Benchmarks Json
         uses: actions/cache@v4
         with:
-          path: benches/main_benchmarks.json
+          path: benches/base.txt
           key: criterion_benchmarks_${{ github.sha }}
 
   Criterion_Compare:
     name: Comparing Micro Benchmarks
     if: "contains(github.event.pull_request.labels.*.name, 'ci: benchmark')"
-    runs-on: benchmarking-runner
+    runs-on: ubuntu-latest
     permissions:
       pull-requests: write
       contents: write
+    env:
+      BENCHER_PROJECT: tailcall-hehe
+      BENCHER_ADAPTER: rust_criterion
+      BENCHER_TESTBED: benchmarking-runner-hehe
+      PR_BENCHMARK_RESULTS: benches/change.txt
+      BASE_BENCHMARK_RESULTS: benches/base.txt
+      # This is the confidence interval for the t-test Threshold
+      # Adjust this value to lower to make the test more sensitive to changes
+      # Adjust this value to higher to make the test less sensitive to changes
+      # https://bencher.dev/docs/explanation/thresholds/#t-test-threshold-upper-boundary
+      UPPER_BOUNDARY: 0.98
     steps:
       - name: Check out code
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Install Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Run Criterion Benchmarks
-        run: |
-          cargo install cargo-criterion rust-script
-          cargo criterion --message-format=json > benches/benchmarks.json
-          ./scripts/json_to_md.rs benches/benchmarks.json > benches/change_benchmarks.md
-
-      - name: Print Criterion Benchmarks
-        run: cat benches/change_benchmarks.md
-
       - name: Restore file
         uses: actions/cache@v4
         with:
-          path: benches/main_benchmarks.json
+          path: benches/base.txt
           key: criterion_benchmarks_${{ github.event.pull_request.base.sha }}
           fail-on-cache-miss: true
 
-      - name: Print Benchmark Comparision
-        run: ./scripts/criterion_compare.rs benches/main_benchmarks.json benches/benchmarks.json table
+      - name: Install Bencher CLI
+        uses: bencherdev/bencher@main
+
+      - name: Track base benchmarks
+        run: |
+          bencher run \
+          --if-branch '${{ github.head_ref }}' \
+          --else-branch \
+          --token "${{ secrets.BENCHER_API_TOKEN }}" \
+          --file "$BASE_BENCHMARK_RESULTS"
 
-      - name: Check Degradation
-        run: ./scripts/criterion_compare.rs  benches/main_benchmarks.json benches/benchmarks.json check
+      - name: Create PR threshold
+        run: |
+          bencher threshold create \
+          --branch '${{ github.head_ref }}' \
+          --testbed "$BENCHER_TESTBED" \
+          --measure latency \
+          --test t \
+          --upper-boundary ${{ env.UPPER_BOUNDARY }} \
+          --token "${{ secrets.BENCHER_API_TOKEN }}"
+
+      - name: Track PR Benchmarks
+        run: |
+          bencher run \
+          --branch '${{ github.head_ref }}' \
+          --ci-number '${{github.event.number}}' \
+          --token "${{ secrets.BENCHER_API_TOKEN }}" \
+          --github-actions "${{ secrets.GITHUB_TOKEN }}" \
+          --err \
+          --file "$PR_BENCHMARK_RESULTS"