From 73300dd3503fbb5cf2b604362216541011b1faf1 Mon Sep 17 00:00:00 2001 From: Julien Lind Date: Tue, 5 Dec 2023 14:04:04 +0100 Subject: [PATCH 01/31] Update pipeline.yml --- .buildkite/pipeline.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8a85b5c17..9354096e4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -89,6 +89,12 @@ steps: - step: "e2e-test" allow_failure: false + - label: "Run go benchmark" + key: "go-benchmark" + command: ".buildkite/scripts/run_benchmark.sh + agents: + provider: "gcp" + - label: ":gcloud: Cloud e2e Test" key: "cloud-e2e-test" command: ".buildkite/scripts/cloud_e2e_test.sh" From f7539b67cca3bda9e4f43a77866d732602323320 Mon Sep 17 00:00:00 2001 From: julienlind Date: Tue, 5 Dec 2023 14:13:32 +0100 Subject: [PATCH 02/31] add initial benchmark for PR --- .buildkite/scripts/run_benchmark.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 .buildkite/scripts/run_benchmark.sh diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh new file mode 100755 index 000000000..7e59d2512 --- /dev/null +++ b/.buildkite/scripts/run_benchmark.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +source .buildkite/scripts/common.sh + +add_bin_path + +with_go + +echo "Starting the go benchmark for the pull request" +BENCH_BASE=next.out make benchmark + +#TODO +#echo "Starting the go benchmark for the base branch" +#BENCH_BASE=base.out make benchmark From 9e725b73a56c8c4c81775a3dd55776807e0665ab Mon Sep 17 00:00:00 2001 From: julienlind Date: Tue, 5 Dec 2023 14:17:14 +0100 Subject: [PATCH 03/31] update pipeline --- .buildkite/pipeline.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 9354096e4..69c6e0318 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -30,7 +30,17 @@ steps: image: "${GO_AGENT_IMAGE}" cpu: "8" memory: "4G" - + + - group: "Performance test" + key: "performance-test" + steps: + - label: "Run go benchmark" + key: "go-benchmark" + command: ".buildkite/scripts/run_benchmark.sh + agents: + provider: "gcp" + depends_on: "check" + - group: "Run tests" key: "tests" steps: @@ -89,12 +99,6 @@ steps: - step: "e2e-test" allow_failure: false - - label: "Run go benchmark" - key: "go-benchmark" - command: ".buildkite/scripts/run_benchmark.sh - agents: - provider: "gcp" - - label: ":gcloud: Cloud e2e Test" key: "cloud-e2e-test" command: ".buildkite/scripts/cloud_e2e_test.sh" From 8d170793f31ba710833b7d5137f5b7efb355591c Mon Sep 17 00:00:00 2001 From: julienlind Date: Tue, 5 Dec 2023 14:19:48 +0100 Subject: [PATCH 04/31] update pipeline --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 69c6e0318..3e965dd86 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -36,11 +36,11 @@ steps: steps: - label: "Run go benchmark" key: "go-benchmark" - command: ".buildkite/scripts/run_benchmark.sh + command: ".buildkite/scripts/run_benchmark.sh" agents: provider: "gcp" depends_on: "check" - + - group: "Run tests" key: "tests" steps: From 0bedb84f8faad1daec190b7e27233994bc64cfaa Mon Sep 17 00:00:00 2001 From: julienlind Date: Tue, 5 Dec 2023 14:40:39 +0100 Subject: [PATCH 05/31] split steps --- .buildkite/pipeline.yml | 13 ++++++++++--- .buildkite/scripts/run_benchmark.sh | 17 ++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3e965dd86..0ec27d215 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -34,9 +34,16 @@ steps: - group: "Performance test" key: "performance-test" steps: - - label: "Run go benchmark" - key: "go-benchmark" - command: ".buildkite/scripts/run_benchmark.sh" + - label: "Run go benchmark for PR branch" + key: "go-benchmark-pr" + command: ".buildkite/scripts/run_benchmark.sh pr" + agents: + provider: "gcp" + + - label: "Run go benchmark for base branch" + key: "go-benchmark-base" + command: ".buildkite/scripts/run_benchmark.sh base" + branches: ${BUILDKITE_BRANCH} agents: provider: "gcp" depends_on: "check" diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 7e59d2512..c31935fdc 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -8,9 +8,16 @@ add_bin_path with_go -echo "Starting the go benchmark for the pull request" -BENCH_BASE=next.out make benchmark +export TYPE=${1} +#export BRANCH="${BUILDKITE_BRANCH}" + +if [[ ${TYPE} == "pr" ]]; then + echo "Starting the go benchmark for the pull request" + BENCH_BASE=next.out make benchmark +fi + +if [[ ${TYPE} == "base" ]]; then + echo "Starting the go benchmark for the pull request" + BENCH_BASE=base.out make benchmark +fi -#TODO -#echo "Starting the go benchmark for the base branch" -#BENCH_BASE=base.out make benchmark From 82bbd237d43eb6ef736c2c5cf41a960ba142fc15 Mon Sep 17 00:00:00 2001 From: Julien Lind Date: Tue, 5 Dec 2023 14:54:53 +0100 Subject: [PATCH 06/31] Update .buildkite/pipeline.yml Co-authored-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0ec27d215..aa875003a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -43,7 +43,7 @@ steps: - label: "Run go benchmark for base branch" key: "go-benchmark-base" command: ".buildkite/scripts/run_benchmark.sh base" - branches: ${BUILDKITE_BRANCH} + branches: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} agents: provider: "gcp" depends_on: "check" From 155a4a53103059b8644e326233554e0e0ad10cb9 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 16:16:46 +0200 Subject: [PATCH 07/31] Add steps for comparing and baseline Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 17 ++++++++++++++++- .buildkite/scripts/run_benchmark.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index aa875003a..27bcfad96 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -37,15 +37,30 @@ steps: - label: "Run go benchmark for PR branch" key: "go-benchmark-pr" command: ".buildkite/scripts/run_benchmark.sh pr" + artifact_paths: + - next.out agents: provider: "gcp" - label: "Run go benchmark for base branch" key: "go-benchmark-base" command: ".buildkite/scripts/run_benchmark.sh base" - branches: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + artifact_paths: + - base.out + agents: + provider: "gcp" + + - label: "Compare results" + key: "go-benchmark-compare" + command: ".buildkite/scripts/run_benchmark.sh compare" + artifact_paths: + - compare.out + depends_on: + - go-benchmark-pr + - go-benchmark-base agents: provider: "gcp" + depends_on: "check" - group: "Run tests" diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index c31935fdc..51ee34b55 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -18,6 +18,35 @@ fi if [[ ${TYPE} == "base" ]]; then echo "Starting the go benchmark for the pull request" + git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} BENCH_BASE=base.out make benchmark fi +if [[ ${TYPE} == "compare" ]]; then + echo "Comparing go benchmarks" + git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + buildkite-agent artifact download "base.out" . + buildkite-agent artifact download "next.out" . + BENCH_BASE=base.out BENCH_NEXT=next.out make benchstat | tee compare.out + + BENCH_NEXT=$(cat base.out) + BENCH_COMPARE=$(cat compare.out) + buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ + ### Benchmark Result +
Benchmark diff against base branch + + ```bash + ${BENCH_COMPARE} + ``` +
+ +
Benchmark result + + ```bash + ${BENCH_NEXT} + ``` +
+_EOF_ +fi + + From 5cccf63a04feea70c5c5ce9065d049fdf885d47c Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 16:24:40 +0200 Subject: [PATCH 08/31] Make each annotate the gobench output Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 2 +- .buildkite/scripts/run_benchmark.sh | 29 ++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 27bcfad96..9902bcd0d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -42,7 +42,7 @@ steps: agents: provider: "gcp" - - label: "Run go benchmark for base branch" + - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" key: "go-benchmark-base" command: ".buildkite/scripts/run_benchmark.sh base" artifact_paths: diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 51ee34b55..2f35aed2e 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -14,12 +14,32 @@ export TYPE=${1} if [[ ${TYPE} == "pr" ]]; then echo "Starting the go benchmark for the pull request" BENCH_BASE=next.out make benchmark + BENCH=$(cat next.out) + buildkite-agent annotate --style 'success' --context "gobench_pr" --append << _EOF_ + ### Benchmark for pull request +
go bench output + + ```bash + ${BENCH} + ``` +
+_EOF_ fi if [[ ${TYPE} == "base" ]]; then echo "Starting the go benchmark for the pull request" git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} BENCH_BASE=base.out make benchmark + BENCH=$(cat base.out) + buildkite-agent annotate --style 'success' --context "gobench_base" --append << _EOF_ + ### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} +
go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + + ```bash + ${BENCH} + ``` +
+_EOF_ fi if [[ ${TYPE} == "compare" ]]; then @@ -28,8 +48,6 @@ if [[ ${TYPE} == "compare" ]]; then buildkite-agent artifact download "base.out" . buildkite-agent artifact download "next.out" . BENCH_BASE=base.out BENCH_NEXT=next.out make benchstat | tee compare.out - - BENCH_NEXT=$(cat base.out) BENCH_COMPARE=$(cat compare.out) buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ ### Benchmark Result @@ -39,13 +57,6 @@ if [[ ${TYPE} == "compare" ]]; then ${BENCH_COMPARE} ``` - -
Benchmark result - - ```bash - ${BENCH_NEXT} - ``` -
_EOF_ fi From 0bc1a952017c0316092e6f485c15bf72720acf53 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 16:27:09 +0200 Subject: [PATCH 09/31] Fix artifact paths Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 6 +++--- .buildkite/scripts/run_benchmark.sh | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 9902bcd0d..a50ae84ad 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -38,7 +38,7 @@ steps: key: "go-benchmark-pr" command: ".buildkite/scripts/run_benchmark.sh pr" artifact_paths: - - next.out + - build/next.out agents: provider: "gcp" @@ -46,7 +46,7 @@ steps: key: "go-benchmark-base" command: ".buildkite/scripts/run_benchmark.sh base" artifact_paths: - - base.out + - build/base.out agents: provider: "gcp" @@ -54,7 +54,7 @@ steps: key: "go-benchmark-compare" command: ".buildkite/scripts/run_benchmark.sh compare" artifact_paths: - - compare.out + - build/compare.out depends_on: - go-benchmark-pr - go-benchmark-base diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 2f35aed2e..2d693628c 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -14,7 +14,7 @@ export TYPE=${1} if [[ ${TYPE} == "pr" ]]; then echo "Starting the go benchmark for the pull request" BENCH_BASE=next.out make benchmark - BENCH=$(cat next.out) + BENCH=$(cat build/next.out) buildkite-agent annotate --style 'success' --context "gobench_pr" --append << _EOF_ ### Benchmark for pull request
go bench output @@ -30,7 +30,7 @@ if [[ ${TYPE} == "base" ]]; then echo "Starting the go benchmark for the pull request" git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} BENCH_BASE=base.out make benchmark - BENCH=$(cat base.out) + BENCH=$(cat build/base.out) buildkite-agent annotate --style 'success' --context "gobench_base" --append << _EOF_ ### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} @@ -45,10 +45,10 @@ fi if [[ ${TYPE} == "compare" ]]; then echo "Comparing go benchmarks" git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - buildkite-agent artifact download "base.out" . - buildkite-agent artifact download "next.out" . - BENCH_BASE=base.out BENCH_NEXT=next.out make benchstat | tee compare.out - BENCH_COMPARE=$(cat compare.out) + buildkite-agent artifact download "build/base.out" . + buildkite-agent artifact download "build/next.out" . + BENCH_BASE=build/base.out BENCH_NEXT=build/next.out make benchstat | tee build/compare.out + BENCH_COMPARE=$(cat build/compare.out) buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ ### Benchmark Result
Benchmark diff against base branch From 885a9896751d6ce995ee99dc130ee3c72ce1282b Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 16:38:51 +0200 Subject: [PATCH 10/31] Compare dont need to checkout base Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 2d693628c..736114a97 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -44,7 +44,6 @@ fi if [[ ${TYPE} == "compare" ]]; then echo "Comparing go benchmarks" - git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} buildkite-agent artifact download "build/base.out" . buildkite-agent artifact download "build/next.out" . BENCH_BASE=build/base.out BENCH_NEXT=build/next.out make benchstat | tee build/compare.out From 5ff8b5de45a2306b45beb8131bb1e7a55f94989e Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 16:39:08 +0200 Subject: [PATCH 11/31] Attempt to increase the cpu and mem for go bench Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a50ae84ad..6d1113c71 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,6 +41,8 @@ steps: - build/next.out agents: provider: "gcp" + cpu: "8" + memory: "4G" - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" key: "go-benchmark-base" @@ -49,6 +51,8 @@ steps: - build/base.out agents: provider: "gcp" + cpu: "8" + memory: "4G" - label: "Compare results" key: "go-benchmark-compare" @@ -60,6 +64,8 @@ steps: - go-benchmark-base agents: provider: "gcp" + cpu: "8" + memory: "4G" depends_on: "check" From a0293924505975a8beffe0ab1c7a5d308d28bb03 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 17:00:13 +0200 Subject: [PATCH 12/31] Escape bash Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 736114a97..80df7b7bc 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -19,9 +19,9 @@ if [[ ${TYPE} == "pr" ]]; then ### Benchmark for pull request
go bench output - ```bash + \`\`\` ${BENCH} - ``` + \`\`\`
_EOF_ fi @@ -35,9 +35,9 @@ if [[ ${TYPE} == "base" ]]; then ### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - ```bash + \`\`\` ${BENCH} - ``` + \`\`\`
_EOF_ fi @@ -52,9 +52,9 @@ if [[ ${TYPE} == "compare" ]]; then ### Benchmark Result
Benchmark diff against base branch - ```bash + \`\`\` ${BENCH_COMPARE} - ``` + \`\`\`
_EOF_ fi From 3501064dda2e6fb0da8eae4fc5f4dce7da5fd0fa Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 17:05:39 +0200 Subject: [PATCH 13/31] Remove the GH action for benchmark Signed-off-by: Alexandros Sapranidis --- .github/workflows/benchmark.yml | 101 -------------------------------- 1 file changed, 101 deletions(-) delete mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 870555236..000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,101 +0,0 @@ -# This script is a modified version of github.com/bool64/dev. -name: benchmark -on: - pull_request: - workflow_dispatch: - inputs: - old: - description: 'Old Ref' - required: false - default: 'main' - new: - description: 'New Ref' - required: true - -# Cancel the workflow in progress in newer build is about to start. -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - GO111MODULE: "on" - RUN_BASE_BENCHMARK: "on" -jobs: - benchmarks: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ (github.event.inputs.new != '') && github.event.inputs.new || github.event.ref }} - - - name: Install Go stable - uses: actions/setup-go@v3 - with: - go-version-file: 'go.mod' - cache: true - - - name: Go cache - uses: actions/cache@v3 - with: - # In order: - # * Module download cache - # * Build cache (Linux) - path: | - ~/go/pkg/mod - key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-cache - - - name: Restore benchstat - uses: actions/cache@v3 - with: - path: ~/go/bin/benchstat - key: ${{ runner.os }}-benchstat-legacy - - - name: Benchmark results for changes - id: bench-next - run: | - BENCH_BASE=next.out make benchmark - echo "next<> $GITHUB_OUTPUT && cat build/next.out >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT - - - name: Benchmark results for base - id: bench-base - if: env.RUN_BASE_BENCHMARK == 'on' && (github.event.pull_request.base.sha != '' || github.event.inputs.old != '') - run: | - git fetch origin main ${{ github.event.pull_request.base.sha }} - HEAD=$(git rev-parse HEAD) - git reset --hard ${{ github.event.pull_request.base.sha }} - BENCH_BASE=base.out make benchmark - git reset --hard $HEAD - echo "base<> $GITHUB_OUTPUT && cat build/base.out >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT - - - name: Benchmark diff against base - id: bench-diff - run: | - echo "${{ steps.bench-next.outputs.next }}" > build/bench-next - echo "${{ steps.bench-base.outputs.base }}" > build/bench-base - BENCH_BASE=bench-base BENCH_NEXT=bench-next make benchstat | tee build/bench-diff.out - echo "diff<> $GITHUB_OUTPUT && cat build/bench-diff.out >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT - - - name: Comment benchmark result - continue-on-error: true - uses: marocchino/sticky-pull-request-comment@v2 - with: - GITHUB_TOKEN: ${{ secrets.PROJECT_ASSIGNER_TOKEN }} - header: benchmark - message: | - ### Benchmark Result -
Benchmark diff against base branch - - ```bash - ${{ steps.bench-diff.outputs.diff }} - ``` -
- -
Benchmark result - - ```bash - ${{ steps.bench-next.outputs.next }} - ``` -
From fd19ba656edda73a2e5d88bb03cdbdc99cbcf65a Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 17:06:48 +0200 Subject: [PATCH 14/31] Increase memory Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6d1113c71..c3d501aff 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -42,7 +42,7 @@ steps: agents: provider: "gcp" cpu: "8" - memory: "4G" + memory: "8G" - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" key: "go-benchmark-base" @@ -52,7 +52,7 @@ steps: agents: provider: "gcp" cpu: "8" - memory: "4G" + memory: "8G" - label: "Compare results" key: "go-benchmark-compare" @@ -65,7 +65,7 @@ steps: agents: provider: "gcp" cpu: "8" - memory: "4G" + memory: "8G" depends_on: "check" From c2384b905d419ea29ec1bbf47dfc933a2d79f8d4 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 17:23:44 +0200 Subject: [PATCH 15/31] Formatting the annotations Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 45 +++++++++++++++++------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 80df7b7bc..8e233279b 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -16,13 +16,15 @@ if [[ ${TYPE} == "pr" ]]; then BENCH_BASE=next.out make benchmark BENCH=$(cat build/next.out) buildkite-agent annotate --style 'success' --context "gobench_pr" --append << _EOF_ - ### Benchmark for pull request -
go bench output +### Benchmark for pull request +
go bench output - \`\`\` - ${BENCH} - \`\`\` -
+\`\`\`bash + +${BENCH} +\`\`\` + +
_EOF_ fi @@ -32,13 +34,15 @@ if [[ ${TYPE} == "base" ]]; then BENCH_BASE=base.out make benchmark BENCH=$(cat build/base.out) buildkite-agent annotate --style 'success' --context "gobench_base" --append << _EOF_ - ### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} -
go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} +### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} +
go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + +\`\`\`bash + +${BENCH} +\`\`\` - \`\`\` - ${BENCH} - \`\`\` -
+
_EOF_ fi @@ -46,16 +50,19 @@ if [[ ${TYPE} == "compare" ]]; then echo "Comparing go benchmarks" buildkite-agent artifact download "build/base.out" . buildkite-agent artifact download "build/next.out" . - BENCH_BASE=build/base.out BENCH_NEXT=build/next.out make benchstat | tee build/compare.out + BENCH_BASE=base.out BENCH_NEXT=next.out make benchstat | tee build/compare.out BENCH_COMPARE=$(cat build/compare.out) buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ - ### Benchmark Result -
Benchmark diff against base branch +### Benchmark Result +
Benchmark diff against base branch + +\`\`\`bash + +${BENCH_COMPARE} + +\`\`\` - \`\`\` - ${BENCH_COMPARE} - \`\`\` -
+
_EOF_ fi From 9caeb6ff752ea03f5e486ca70a9f57fd87c7d7a7 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 5 Dec 2023 17:25:00 +0200 Subject: [PATCH 16/31] Change the type of the VMs for benchmarks Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c3d501aff..8b255f592 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,8 +41,7 @@ steps: - build/next.out agents: provider: "gcp" - cpu: "8" - memory: "8G" + machineType: "c2-standard-8" - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" key: "go-benchmark-base" @@ -51,8 +50,7 @@ steps: - build/base.out agents: provider: "gcp" - cpu: "8" - memory: "8G" + machineType: "c2-standard-8" - label: "Compare results" key: "go-benchmark-compare" @@ -64,8 +62,6 @@ steps: - go-benchmark-base agents: provider: "gcp" - cpu: "8" - memory: "8G" depends_on: "check" From 05165ade99beea62910b10877b3e8122cb1ff036 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 09:33:59 +0200 Subject: [PATCH 17/31] Ignore direnv files Signed-off-by: Alexandros Sapranidis --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 54e2faa6a..cf5a3e2bc 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,7 @@ dev-tools/cloud/terraform/*.tfvars* .service_token* .kibana_service_token + + +# direnv +.envrc* \ No newline at end of file From b684303b2d40ff1d02a83442a30e78ead32d6fee Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 11:57:37 +0200 Subject: [PATCH 18/31] Parse go benchmark results with gobenchdata Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 18 ++++++++++-------- gobenchdata-checks.yml | 11 +++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100755 gobenchdata-checks.yml diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 8e233279b..f4a3f6b2b 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -48,20 +48,22 @@ fi if [[ ${TYPE} == "compare" ]]; then echo "Comparing go benchmarks" + go install go.bobheadxi.dev/gobenchdata@latest buildkite-agent artifact download "build/base.out" . buildkite-agent artifact download "build/next.out" . - BENCH_BASE=base.out BENCH_NEXT=next.out make benchstat | tee build/compare.out - BENCH_COMPARE=$(cat build/compare.out) - buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ -### Benchmark Result -
Benchmark diff against base branch -\`\`\`bash + cat build/base.out| gobenchdata --json build/base.out + cat build/next.out| gobenchdata --json build/next.out + gobenchdata checks eval base.json next.json --json build/full_report.json + cat build/full_report.json| jq 'del(.Checks."My Check".Diffs[]| select(.Status == "pass") )'| tee build/failed_report.json + gobenchdata checks report failed.json | tee build/failed_summary.md + BENCH_COMPARE=$(cat build/failed_summary.md) + buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ +### Benchmark comparison +
Comparison table of benchmark results of HEAD compared to ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} ${BENCH_COMPARE} -\`\`\` -
_EOF_ fi diff --git a/gobenchdata-checks.yml b/gobenchdata-checks.yml new file mode 100755 index 000000000..77b32b1b5 --- /dev/null +++ b/gobenchdata-checks.yml @@ -0,0 +1,11 @@ +checks: + - name: My Check + description: |- + Define a check here - in this example, we caculate % difference for NsPerOp in the diff function. + diff is a function where you receive two parameters, current and base, and in general this function + should return a negative value for an improvement and a positive value for a regression. + package: . + benchmarks: [] + diff: (current.NsPerOp - base.NsPerOp) / base.NsPerOp * 100 + thresholds: + max: 10 From 4aeec7afe7bb3daefdd6cc603e2931a3636ea406 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 11:59:39 +0200 Subject: [PATCH 19/31] Fix the artifact path Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8b255f592..2b947fa52 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -56,7 +56,9 @@ steps: key: "go-benchmark-compare" command: ".buildkite/scripts/run_benchmark.sh compare" artifact_paths: - - build/compare.out + - build/failed_summary.md + - build/failed_report.json + - build/full_report.json depends_on: - go-benchmark-pr - go-benchmark-base From 3a1d18eaba66197e55b21e99db3c26832c4a6745 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 12:14:11 +0200 Subject: [PATCH 20/31] fix the filepaths Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index f4a3f6b2b..30141b67b 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -52,8 +52,8 @@ if [[ ${TYPE} == "compare" ]]; then buildkite-agent artifact download "build/base.out" . buildkite-agent artifact download "build/next.out" . - cat build/base.out| gobenchdata --json build/base.out - cat build/next.out| gobenchdata --json build/next.out + cat build/base.out| gobenchdata --json build/base.json + cat build/next.out| gobenchdata --json build/next.json gobenchdata checks eval base.json next.json --json build/full_report.json cat build/full_report.json| jq 'del(.Checks."My Check".Diffs[]| select(.Status == "pass") )'| tee build/failed_report.json gobenchdata checks report failed.json | tee build/failed_summary.md From aaa85ed6b64653bcdb72d50f4d279c5fd0c92ae3 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 12:28:58 +0200 Subject: [PATCH 21/31] Fix the path again Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 30141b67b..e12182518 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -54,7 +54,7 @@ if [[ ${TYPE} == "compare" ]]; then cat build/base.out| gobenchdata --json build/base.json cat build/next.out| gobenchdata --json build/next.json - gobenchdata checks eval base.json next.json --json build/full_report.json + gobenchdata checks eval build/base.json build/next.json --json build/full_report.json cat build/full_report.json| jq 'del(.Checks."My Check".Diffs[]| select(.Status == "pass") )'| tee build/failed_report.json gobenchdata checks report failed.json | tee build/failed_summary.md BENCH_COMPARE=$(cat build/failed_summary.md) From 41ec3c84c8b7ea7b4d0d3af5607068d64fcce2bd Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 12:51:53 +0200 Subject: [PATCH 22/31] Make annotation reflect status Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index e12182518..b2c2c1d47 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -16,7 +16,7 @@ if [[ ${TYPE} == "pr" ]]; then BENCH_BASE=next.out make benchmark BENCH=$(cat build/next.out) buildkite-agent annotate --style 'success' --context "gobench_pr" --append << _EOF_ -### Benchmark for pull request +#### Benchmark for pull request
go bench output \`\`\`bash @@ -34,7 +34,7 @@ if [[ ${TYPE} == "base" ]]; then BENCH_BASE=base.out make benchmark BENCH=$(cat build/base.out) buildkite-agent annotate --style 'success' --context "gobench_base" --append << _EOF_ -### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} +#### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} \`\`\`bash @@ -55,17 +55,30 @@ if [[ ${TYPE} == "compare" ]]; then cat build/base.out| gobenchdata --json build/base.json cat build/next.out| gobenchdata --json build/next.json gobenchdata checks eval build/base.json build/next.json --json build/full_report.json - cat build/full_report.json| jq 'del(.Checks."My Check".Diffs[]| select(.Status == "pass") )'| tee build/failed_report.json - gobenchdata checks report failed.json | tee build/failed_summary.md - BENCH_COMPARE=$(cat build/failed_summary.md) - buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ -### Benchmark comparison + status=$(jq -r '.Status' build/full_report.json) + if [[ $status == "fail" ]]; then + cat build/full_report.json| jq 'del(.Checks."My Check".Diffs[]| select(.Status == "pass") )'| tee build/failed_report.json + gobenchdata checks report failed.json | tee build/failed_summary.md + BENCH_COMPARE=$(cat build/failed_summary.md) + buildkite-agent annotate --style 'error' --context "benchstat" --append << _EOF_ +#### Benchmark comparison
Comparison table of benchmark results of HEAD compared to ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} ${BENCH_COMPARE}
_EOF_ + else + BENCH_COMPARE=$(gobenchdata checks report build/full_report.json) + buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ +#### Benchmark comparison +
No significant performance issue detect against ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + +${BENCH_COMPARE} + +
+_EOF_ + fi fi From 8f539584973599c544b334218d8f71b6027647e9 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 13:51:17 +0200 Subject: [PATCH 23/31] Fix failed json path Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index b2c2c1d47..ea37b2c55 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -58,7 +58,7 @@ if [[ ${TYPE} == "compare" ]]; then status=$(jq -r '.Status' build/full_report.json) if [[ $status == "fail" ]]; then cat build/full_report.json| jq 'del(.Checks."My Check".Diffs[]| select(.Status == "pass") )'| tee build/failed_report.json - gobenchdata checks report failed.json | tee build/failed_summary.md + gobenchdata checks report build/failed_report.json | tee build/failed_summary.md BENCH_COMPARE=$(cat build/failed_summary.md) buildkite-agent annotate --style 'error' --context "benchstat" --append << _EOF_ #### Benchmark comparison From 12c055056e7ace40d50b82ed1416aabd00a98907 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 14:08:01 +0200 Subject: [PATCH 24/31] Suppress gobenchdata error exiting Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index ea37b2c55..da5cb5bed 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -54,6 +54,7 @@ if [[ ${TYPE} == "compare" ]]; then cat build/base.out| gobenchdata --json build/base.json cat build/next.out| gobenchdata --json build/next.json + set +e # suppress error handling of gobenchdata gobenchdata checks eval build/base.json build/next.json --json build/full_report.json status=$(jq -r '.Status' build/full_report.json) if [[ $status == "fail" ]]; then @@ -68,6 +69,7 @@ ${BENCH_COMPARE}
_EOF_ + exit 1 else BENCH_COMPARE=$(gobenchdata checks report build/full_report.json) buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ From 50874ef0a4a719001d67c1384fe4937a3c08ccb5 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 14:56:30 +0200 Subject: [PATCH 25/31] Enable mem reporting for gobench by default Signed-off-by: Alexandros Sapranidis --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f55035f88..0db7f090a 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ BUILDER_IMAGE=docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-d #Benchmark related targets BENCH_BASE ?= benchmark-$(COMMIT).out BENCH_NEXT ?= -BENCHMARK_ARGS := -count=8 +BENCHMARK_ARGS := -count=8 -benchmem BENCHMARK_PACKAGE ?= ./... BENCHMARK_FILTER ?= Bench From b0e203a13f35ae30d3276d6b75175172178a9316 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 14:57:47 +0200 Subject: [PATCH 26/31] Add links to artifact download in bk annotation Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index da5cb5bed..8294cdc00 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -68,8 +68,10 @@ if [[ ${TYPE} == "compare" ]]; then ${BENCH_COMPARE}
+ +Download failed_summary.md , full_report.json _EOF_ - exit 1 + exit 1 # fail the build if the status is fail else BENCH_COMPARE=$(gobenchdata checks report build/full_report.json) buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ @@ -79,6 +81,8 @@ _EOF_ ${BENCH_COMPARE}
+ +Download full_report.json _EOF_ fi fi From 9846de7ca15793f820f287ebbfce43ff44b9890e Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 16:06:58 +0200 Subject: [PATCH 27/31] change the gobenchdata check name Signed-off-by: Alexandros Sapranidis --- gobenchdata-checks.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gobenchdata-checks.yml b/gobenchdata-checks.yml index 77b32b1b5..4354f6ed9 100755 --- a/gobenchdata-checks.yml +++ b/gobenchdata-checks.yml @@ -1,9 +1,7 @@ checks: - - name: My Check + - name: timePerOp description: |- - Define a check here - in this example, we caculate % difference for NsPerOp in the diff function. - diff is a function where you receive two parameters, current and base, and in general this function - should return a negative value for an improvement and a positive value for a regression. + This check is set to fail when there are benchmark tests that are slower than the defined threshold package: . benchmarks: [] diff: (current.NsPerOp - base.NsPerOp) / base.NsPerOp * 100 From 45b5dd54826da50e75669c7b98ac57faddac93e9 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 16:07:52 +0200 Subject: [PATCH 28/31] Improve the benchmark script for the step Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 8294cdc00..72b2571fb 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -10,21 +10,23 @@ with_go export TYPE=${1} #export BRANCH="${BUILDKITE_BRANCH}" +export BENCHMARK_ARGS="-count=8 -benchmem" if [[ ${TYPE} == "pr" ]]; then echo "Starting the go benchmark for the pull request" BENCH_BASE=next.out make benchmark BENCH=$(cat build/next.out) - buildkite-agent annotate --style 'success' --context "gobench_pr" --append << _EOF_ + buildkite-agent annotate --style 'info' --context "gobench_pr" --append << _EOF_ #### Benchmark for pull request
go bench output \`\`\`bash - ${BENCH} \`\`\`
+ +Download next.out _EOF_ fi @@ -33,16 +35,17 @@ if [[ ${TYPE} == "base" ]]; then git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} BENCH_BASE=base.out make benchmark BENCH=$(cat build/base.out) - buildkite-agent annotate --style 'success' --context "gobench_base" --append << _EOF_ + buildkite-agent annotate --style 'info' --context "gobench_base" --append << _EOF_ #### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} \`\`\`bash - ${BENCH} \`\`\`
+ +Download ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}.out _EOF_ fi @@ -58,7 +61,9 @@ if [[ ${TYPE} == "compare" ]]; then gobenchdata checks eval build/base.json build/next.json --json build/full_report.json status=$(jq -r '.Status' build/full_report.json) if [[ $status == "fail" ]]; then - cat build/full_report.json| jq 'del(.Checks."My Check".Diffs[]| select(.Status == "pass") )'| tee build/failed_report.json + cat build/full_report.json| \ + jq 'del(.Checks.timePerOp.Diffs[]| select(.Status == "pass") )'| \ + tee build/failed_report.json gobenchdata checks report build/failed_report.json | tee build/failed_summary.md BENCH_COMPARE=$(cat build/failed_summary.md) buildkite-agent annotate --style 'error' --context "benchstat" --append << _EOF_ From 2aa34e98cacc0c144e6a67ed9ab51fc2ae042d52 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 16:34:55 +0200 Subject: [PATCH 29/31] Allow gobenchdata failure to not block Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/run_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh index 72b2571fb..cb5150b93 100755 --- a/.buildkite/scripts/run_benchmark.sh +++ b/.buildkite/scripts/run_benchmark.sh @@ -76,7 +76,7 @@ ${BENCH_COMPARE} Download failed_summary.md , full_report.json _EOF_ - exit 1 # fail the build if the status is fail + #exit 1 # fail the build if the status is fail else BENCH_COMPARE=$(gobenchdata checks report build/full_report.json) buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ From 69a5cf7e9dad470cd74a3ddf7f350d6e22525ad5 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 16:36:30 +0200 Subject: [PATCH 30/31] Reset Benchmark timer for Bulk Attempt to fix the reporting of time of go benchmark Signed-off-by: Alexandros Sapranidis --- internal/pkg/bulk/bulk_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go index 2dfd1b9db..13f81a5c8 100644 --- a/internal/pkg/bulk/bulk_test.go +++ b/internal/pkg/bulk/bulk_test.go @@ -339,6 +339,8 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) { go func(sampleData []byte) { defer wait.Done() + b.ResetTimer() + b.StartTimer() for j := 0; j < b.N; j++ { // Create id, err := bulker.Create(ctx, index, "", sampleData) @@ -364,6 +366,7 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) { b.Error(err) } } + b.StopTimer() }(samples[i]) } @@ -373,7 +376,7 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) { } func BenchmarkMockBulk(b *testing.B) { - + b.StopTimer() benchmarks := []int{1, 8, 64, 4096, 32768} // Create the samples outside the loop to avoid accounting From 2e8253d71d3f3177cd5016d71de887d47be084a4 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 6 Dec 2023 16:52:56 +0200 Subject: [PATCH 31/31] Revert "Reset Benchmark timer for Bulk" This reverts commit 69a5cf7e9dad470cd74a3ddf7f350d6e22525ad5. --- internal/pkg/bulk/bulk_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go index 13f81a5c8..2dfd1b9db 100644 --- a/internal/pkg/bulk/bulk_test.go +++ b/internal/pkg/bulk/bulk_test.go @@ -339,8 +339,6 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) { go func(sampleData []byte) { defer wait.Done() - b.ResetTimer() - b.StartTimer() for j := 0; j < b.N; j++ { // Create id, err := bulker.Create(ctx, index, "", sampleData) @@ -366,7 +364,6 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) { b.Error(err) } } - b.StopTimer() }(samples[i]) } @@ -376,7 +373,7 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) { } func BenchmarkMockBulk(b *testing.B) { - b.StopTimer() + benchmarks := []int{1, 8, 64, 4096, 32768} // Create the samples outside the loop to avoid accounting