diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8a85b5c17..2b947fa52 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -30,6 +30,42 @@ steps: image: "${GO_AGENT_IMAGE}" cpu: "8" memory: "4G" + + - group: "Performance test" + key: "performance-test" + steps: + - label: "Run go benchmark for PR branch" + key: "go-benchmark-pr" + command: ".buildkite/scripts/run_benchmark.sh pr" + artifact_paths: + - build/next.out + agents: + provider: "gcp" + machineType: "c2-standard-8" + + - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" + key: "go-benchmark-base" + command: ".buildkite/scripts/run_benchmark.sh base" + artifact_paths: + - build/base.out + agents: + provider: "gcp" + machineType: "c2-standard-8" + + - label: "Compare results" + key: "go-benchmark-compare" + command: ".buildkite/scripts/run_benchmark.sh compare" + artifact_paths: + - build/failed_summary.md + - build/failed_report.json + - build/full_report.json + depends_on: + - go-benchmark-pr + - go-benchmark-base + agents: + provider: "gcp" + + depends_on: "check" - group: "Run tests" key: "tests" diff --git a/.buildkite/scripts/run_benchmark.sh b/.buildkite/scripts/run_benchmark.sh new file mode 100755 index 000000000..cb5150b93 --- /dev/null +++ b/.buildkite/scripts/run_benchmark.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -euo pipefail + +source .buildkite/scripts/common.sh + +add_bin_path + +with_go + +export TYPE=${1} +#export BRANCH="${BUILDKITE_BRANCH}" +export BENCHMARK_ARGS="-count=8 -benchmem" + +if [[ ${TYPE} == "pr" ]]; then + echo "Starting the go benchmark for the pull request" + BENCH_BASE=next.out make benchmark + BENCH=$(cat build/next.out) + buildkite-agent annotate --style 'info' --context "gobench_pr" --append << _EOF_ +#### Benchmark for pull request +
go bench output + +\`\`\`bash +${BENCH} +\`\`\` + +
+ +Download next.out +_EOF_ +fi + +if [[ ${TYPE} == "base" ]]; then + echo "Starting the go benchmark for the pull request" + git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + BENCH_BASE=base.out make benchmark + BENCH=$(cat build/base.out) + buildkite-agent annotate --style 'info' --context "gobench_base" --append << _EOF_ +#### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} +
go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + +\`\`\`bash +${BENCH} +\`\`\` + +
+ +Download ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}.out +_EOF_ +fi + +if [[ ${TYPE} == "compare" ]]; then + echo "Comparing go benchmarks" + go install go.bobheadxi.dev/gobenchdata@latest + buildkite-agent artifact download "build/base.out" . + buildkite-agent artifact download "build/next.out" . + + cat build/base.out| gobenchdata --json build/base.json + cat build/next.out| gobenchdata --json build/next.json + set +e # suppress error handling of gobenchdata + gobenchdata checks eval build/base.json build/next.json --json build/full_report.json + status=$(jq -r '.Status' build/full_report.json) + if [[ $status == "fail" ]]; then + cat build/full_report.json| \ + jq 'del(.Checks.timePerOp.Diffs[]| select(.Status == "pass") )'| \ + tee build/failed_report.json + gobenchdata checks report build/failed_report.json | tee build/failed_summary.md + BENCH_COMPARE=$(cat build/failed_summary.md) + buildkite-agent annotate --style 'error' --context "benchstat" --append << _EOF_ +#### Benchmark comparison +
Comparison table of benchmark results of HEAD compared to ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + +${BENCH_COMPARE} + +
+ +Download failed_summary.md , full_report.json +_EOF_ + #exit 1 # fail the build if the status is fail + else + BENCH_COMPARE=$(gobenchdata checks report build/full_report.json) + buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_ +#### Benchmark comparison +
No significant performance issue detect against ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + +${BENCH_COMPARE} + +
+ +Download full_report.json +_EOF_ + fi +fi + + diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 870555236..000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,101 +0,0 @@ -# This script is a modified version of github.com/bool64/dev. -name: benchmark -on: - pull_request: - workflow_dispatch: - inputs: - old: - description: 'Old Ref' - required: false - default: 'main' - new: - description: 'New Ref' - required: true - -# Cancel the workflow in progress in newer build is about to start. -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - GO111MODULE: "on" - RUN_BASE_BENCHMARK: "on" -jobs: - benchmarks: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ (github.event.inputs.new != '') && github.event.inputs.new || github.event.ref }} - - - name: Install Go stable - uses: actions/setup-go@v3 - with: - go-version-file: 'go.mod' - cache: true - - - name: Go cache - uses: actions/cache@v3 - with: - # In order: - # * Module download cache - # * Build cache (Linux) - path: | - ~/go/pkg/mod - key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-cache - - - name: Restore benchstat - uses: actions/cache@v3 - with: - path: ~/go/bin/benchstat - key: ${{ runner.os }}-benchstat-legacy - - - name: Benchmark results for changes - id: bench-next - run: | - BENCH_BASE=next.out make benchmark - echo "next<> $GITHUB_OUTPUT && cat build/next.out >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT - - - name: Benchmark results for base - id: bench-base - if: env.RUN_BASE_BENCHMARK == 'on' && (github.event.pull_request.base.sha != '' || github.event.inputs.old != '') - run: | - git fetch origin main ${{ github.event.pull_request.base.sha }} - HEAD=$(git rev-parse HEAD) - git reset --hard ${{ github.event.pull_request.base.sha }} - BENCH_BASE=base.out make benchmark - git reset --hard $HEAD - echo "base<> $GITHUB_OUTPUT && cat build/base.out >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT - - - name: Benchmark diff against base - id: bench-diff - run: | - echo "${{ steps.bench-next.outputs.next }}" > build/bench-next - echo "${{ steps.bench-base.outputs.base }}" > build/bench-base - BENCH_BASE=bench-base BENCH_NEXT=bench-next make benchstat | tee build/bench-diff.out - echo "diff<> $GITHUB_OUTPUT && cat build/bench-diff.out >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT - - - name: Comment benchmark result - continue-on-error: true - uses: marocchino/sticky-pull-request-comment@v2 - with: - GITHUB_TOKEN: ${{ secrets.PROJECT_ASSIGNER_TOKEN }} - header: benchmark - message: | - ### Benchmark Result -
Benchmark diff against base branch - - ```bash - ${{ steps.bench-diff.outputs.diff }} - ``` -
- -
Benchmark result - - ```bash - ${{ steps.bench-next.outputs.next }} - ``` -
diff --git a/.gitignore b/.gitignore index 54e2faa6a..cf5a3e2bc 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,7 @@ dev-tools/cloud/terraform/*.tfvars* .service_token* .kibana_service_token + + +# direnv +.envrc* \ No newline at end of file diff --git a/Makefile b/Makefile index f55035f88..0db7f090a 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ BUILDER_IMAGE=docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-d #Benchmark related targets BENCH_BASE ?= benchmark-$(COMMIT).out BENCH_NEXT ?= -BENCHMARK_ARGS := -count=8 +BENCHMARK_ARGS := -count=8 -benchmem BENCHMARK_PACKAGE ?= ./... BENCHMARK_FILTER ?= Bench diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 1f4a7a3de..f515930b2 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.12.0-9d443b17-SNAPSHOT +ELASTICSEARCH_VERSION=8.12.0-9bbde39d-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 diff --git a/gobenchdata-checks.yml b/gobenchdata-checks.yml new file mode 100755 index 000000000..4354f6ed9 --- /dev/null +++ b/gobenchdata-checks.yml @@ -0,0 +1,9 @@ +checks: + - name: timePerOp + description: |- + This check is set to fail when there are benchmark tests that are slower than the defined threshold + package: . + benchmarks: [] + diff: (current.NsPerOp - base.NsPerOp) / base.NsPerOp * 100 + thresholds: + max: 10 diff --git a/version/version.go b/version/version.go index 9c1c789a0..d2191e636 100644 --- a/version/version.go +++ b/version/version.go @@ -6,4 +6,4 @@ package version // DefaultVersion is the current release version of Fleet-server, this version must match the // Elastic Agent version. -const DefaultVersion = "8.12.0" +const DefaultVersion = "8.13.0"