From f9918e0f19ddc006789daf0e4f92fcc096255faa Mon Sep 17 00:00:00 2001 From: alankritdabral <95607957+alankritdabral@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:12:20 +0530 Subject: [PATCH 1/8] Create track_benchmark.yml --- .github/workflows/track_benchmark.yml | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/track_benchmark.yml diff --git a/.github/workflows/track_benchmark.yml b/.github/workflows/track_benchmark.yml new file mode 100644 index 0000000000..f4be7200e6 --- /dev/null +++ b/.github/workflows/track_benchmark.yml @@ -0,0 +1,45 @@ +name: Track Benchmark + +on: + push: + paths-ignore: ["docs/**", "**.md"] + branches: + - main + pull_request: + paths-ignore: ["docs/**", "**.md"] + +jobs: + track_benchmark_with_bencher: + name: Track Benchmarks with Bencher + permissions: + pull-requests: write + contents: write + runs-on: ubuntu-latest + env: + BENCHER_PROJECT: tailcall-2192953494 + BENCHER_TESTBED: ubuntu-latest + BENCHER_ADAPTER: json + BASE_BENCHMARK_RESULTS: benches/track_benchmark.json + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Convert Cache into Proper Json + run: | + cat benches/main_benchmarks.json | jq -s 'map(select(.id and .mean and .slope and .median) | {id: .id, mean: .mean, slope: .slope, median: .median})' > benches/track_benchmark.json + - name: Install Bencher CLI + uses: bencherdev/bencher@main + + - name: Track base Benchmarks + run: | + bencher run \ + --if-branch '${{ github.event.pull_request.head.ref }}' \ + --else-if-branch '${{ github.event.pull_request.base.ref }}' \ + --else-if-branch main \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + --ci-id '${{ github.event.pull_request.number }}' \ + --github-actions "${{ secrets.GITHUB_TOKEN }}" \ + --err \ + --file "$BASE_BENCHMARK_RESULTS" From e969b1206a18e325cdbbb385ea686e7cefb05461 Mon Sep 17 00:00:00 2001 From: alankritdabral <95607957+alankritdabral@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:13:38 +0530 Subject: [PATCH 2/8] Update track_benchmark.yml --- .github/workflows/track_benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/track_benchmark.yml b/.github/workflows/track_benchmark.yml index f4be7200e6..e6d62774dd 100644 --- a/.github/workflows/track_benchmark.yml +++ b/.github/workflows/track_benchmark.yml @@ -38,7 +38,7 @@ jobs: --if-branch '${{ github.event.pull_request.head.ref }}' \ --else-if-branch '${{ github.event.pull_request.base.ref }}' \ --else-if-branch main \ - --token "${{ secrets.BENCHER_API_TOKEN }}" \ + --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxNzIwMDYyMTI4LCJpYXQiOjE3MTAwNjIxMjgsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.0QUtYqWgPtyYV8d6SsK2m4hNoa88r6aSI6Ojupatd3s \ --ci-id '${{ github.event.pull_request.number }}' \ --github-actions "${{ secrets.GITHUB_TOKEN }}" \ --err \ From 013258009da278e15089a8c1e1a9d8ede95989b1 Mon Sep 17 00:00:00 2001 From: alankritdabral <95607957+alankritdabral@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:14:38 +0530 Subject: [PATCH 3/8] Update track_benchmark.yml --- .github/workflows/track_benchmark.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/track_benchmark.yml b/.github/workflows/track_benchmark.yml index e6d62774dd..c7f1f69953 100644 --- a/.github/workflows/track_benchmark.yml +++ b/.github/workflows/track_benchmark.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest env: BENCHER_PROJECT: tailcall-2192953494 - BENCHER_TESTBED: ubuntu-latest + # BENCHER_TESTBED: ubuntu-latest BENCHER_ADAPTER: json BASE_BENCHMARK_RESULTS: benches/track_benchmark.json steps: @@ -35,9 +35,9 @@ jobs: - name: Track base Benchmarks run: | bencher run \ - --if-branch '${{ github.event.pull_request.head.ref }}' \ - --else-if-branch '${{ github.event.pull_request.base.ref }}' \ - --else-if-branch main \ + # --if-branch '${{ github.event.pull_request.head.ref }}' \ + # --else-if-branch '${{ github.event.pull_request.base.ref }}' \ + # --else-if-branch main \ --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxNzIwMDYyMTI4LCJpYXQiOjE3MTAwNjIxMjgsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.0QUtYqWgPtyYV8d6SsK2m4hNoa88r6aSI6Ojupatd3s \ --ci-id '${{ github.event.pull_request.number }}' \ --github-actions "${{ secrets.GITHUB_TOKEN }}" \ From 19fc4fadffbee965480d84be1914c4172dcb5971 Mon Sep 17 00:00:00 2001 From: alankritdabral <95607957+alankritdabral@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:15:42 +0530 Subject: [PATCH 4/8] Update track_benchmark.yml --- .github/workflows/track_benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/track_benchmark.yml b/.github/workflows/track_benchmark.yml index c7f1f69953..b0942ca8b9 100644 --- a/.github/workflows/track_benchmark.yml +++ b/.github/workflows/track_benchmark.yml @@ -35,11 +35,11 @@ jobs: - name: Track base Benchmarks run: | bencher run \ - # --if-branch '${{ github.event.pull_request.head.ref }}' \ - # --else-if-branch '${{ github.event.pull_request.base.ref }}' \ - # --else-if-branch main \ --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxNzIwMDYyMTI4LCJpYXQiOjE3MTAwNjIxMjgsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.0QUtYqWgPtyYV8d6SsK2m4hNoa88r6aSI6Ojupatd3s \ --ci-id '${{ github.event.pull_request.number }}' \ --github-actions "${{ secrets.GITHUB_TOKEN }}" \ --err \ --file "$BASE_BENCHMARK_RESULTS" + # --if-branch '${{ github.event.pull_request.head.ref }}' \ + # --else-if-branch '${{ github.event.pull_request.base.ref }}' \ + # --else-if-branch main \ From 8aa32d4ab281e6bb70192ee2b193ec07c978f639 Mon Sep 17 00:00:00 2001 From: alankritdabral <95607957+alankritdabral@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:24:54 +0530 Subject: [PATCH 5/8] Update track_benchmark.yml --- .github/workflows/track_benchmark.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/track_benchmark.yml b/.github/workflows/track_benchmark.yml index b0942ca8b9..1774a973e1 100644 --- a/.github/workflows/track_benchmark.yml +++ b/.github/workflows/track_benchmark.yml @@ -17,18 +17,17 @@ jobs: runs-on: ubuntu-latest env: BENCHER_PROJECT: tailcall-2192953494 + BENCHER_ADAPTER: rust_criterion # BENCHER_TESTBED: ubuntu-latest - BENCHER_ADAPTER: json - BASE_BENCHMARK_RESULTS: benches/track_benchmark.json steps: - name: Check out code uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Convert Cache into Proper Json - run: | - cat benches/main_benchmarks.json | jq -s 'map(select(.id and .mean and .slope and .median) | {id: .id, mean: .mean, slope: .slope, median: .median})' > benches/track_benchmark.json + - name: Install rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install Bencher CLI uses: bencherdev/bencher@main @@ -36,10 +35,4 @@ jobs: run: | bencher run \ --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxNzIwMDYyMTI4LCJpYXQiOjE3MTAwNjIxMjgsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.0QUtYqWgPtyYV8d6SsK2m4hNoa88r6aSI6Ojupatd3s \ - --ci-id '${{ github.event.pull_request.number }}' \ - --github-actions "${{ secrets.GITHUB_TOKEN }}" \ - --err \ - --file "$BASE_BENCHMARK_RESULTS" - # --if-branch '${{ github.event.pull_request.head.ref }}' \ - # --else-if-branch '${{ github.event.pull_request.base.ref }}' \ - # --else-if-branch main \ + 'cargo +stable bench' From cc876498077d428a90d73488191d8bc81a459d0c Mon Sep 17 00:00:00 2001 From: alankritdabral <95607957+alankritdabral@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:29:41 +0530 Subject: [PATCH 6/8] Update track_benchmark.yml --- .github/workflows/track_benchmark.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/track_benchmark.yml b/.github/workflows/track_benchmark.yml index 1774a973e1..c475a545ac 100644 --- a/.github/workflows/track_benchmark.yml +++ b/.github/workflows/track_benchmark.yml @@ -28,6 +28,10 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Install Criterion + run: | + cargo install cargo-criterion + - name: Install Bencher CLI uses: bencherdev/bencher@main @@ -35,4 +39,4 @@ jobs: run: | bencher run \ --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxNzIwMDYyMTI4LCJpYXQiOjE3MTAwNjIxMjgsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.0QUtYqWgPtyYV8d6SsK2m4hNoa88r6aSI6Ojupatd3s \ - 'cargo +stable bench' + 'cargo criterion' From cc0af6abff2cb27259f627c2b4976fa6f414e55e Mon Sep 17 00:00:00 2001 From: alankritdabral <95607957+alankritdabral@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:53:14 +0530 Subject: [PATCH 7/8] Update track_benchmark.yml --- .github/workflows/track_benchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/track_benchmark.yml b/.github/workflows/track_benchmark.yml index c475a545ac..1f0018d95b 100644 --- a/.github/workflows/track_benchmark.yml +++ b/.github/workflows/track_benchmark.yml @@ -17,8 +17,7 @@ jobs: runs-on: ubuntu-latest env: BENCHER_PROJECT: tailcall-2192953494 - BENCHER_ADAPTER: rust_criterion - # BENCHER_TESTBED: ubuntu-latest + BENCHER_ADAPTER: json steps: - name: Check out code uses: actions/checkout@v4 From ba7bc72dd24cacc851b06314b67872813afee6ff Mon Sep 17 00:00:00 2001 From: alankritdabral <95607957+alankritdabral@users.noreply.github.com> Date: Mon, 11 Mar 2024 21:09:51 +0530 Subject: [PATCH 8/8] Update track_benchmark.yml --- .github/workflows/track_benchmark.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/track_benchmark.yml b/.github/workflows/track_benchmark.yml index 1f0018d95b..b8bd186a3d 100644 --- a/.github/workflows/track_benchmark.yml +++ b/.github/workflows/track_benchmark.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest env: BENCHER_PROJECT: tailcall-2192953494 - BENCHER_ADAPTER: json + BENCHER_ADAPTER: rust_criterion steps: - name: Check out code uses: actions/checkout@v4 @@ -27,10 +27,6 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable - - name: Install Criterion - run: | - cargo install cargo-criterion - - name: Install Bencher CLI uses: bencherdev/bencher@main @@ -38,4 +34,4 @@ jobs: run: | bencher run \ --token eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjoxNzIwMDYyMTI4LCJpYXQiOjE3MTAwNjIxMjgsImlzcyI6ImJlbmNoZXIuZGV2Iiwic3ViIjoiZGFicmFsYWxhbmtyaXRAZ21haWwuY29tIiwib3JnIjpudWxsfQ.0QUtYqWgPtyYV8d6SsK2m4hNoa88r6aSI6Ojupatd3s \ - 'cargo criterion' + 'cargo bench'