Merge pull request #200 from miparnisari/run-benchmarks

ci: run benchmarks on every pr and alert if perf is worse

.github/workflows/master.yaml

@@ -0,0 +1,48 @@
+name: Master
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: read
+
+jobs:
+  go-bench:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: './go.mod'
+          cache-dependency-path: './go.sum'
+          check-latest: true
+
+      - name: Run benchmark and store the output to a file
+        run: |
+          set -o pipefail
+          make bench | tee bench_output.txt
+
+      - name: Get benchmark as JSON
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: bench_output.txt
+          # Write benchmarks to this file
+          external-data-json-path: ./cache/benchmark-data.json
+          # Workflow will fail when an alert happens
+          fail-on-alert: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+
+      - name: Save benchmark JSON to cache
+        uses: actions/cache/save@v3
+        with:
+          path: ./cache/benchmark-data.json
+          # Save with commit hash to avoid "cache already exists"
+          # Save with OS to prevent comparing against results from different CPUs
+          key: ${{ github.sha }}-${{ runner.os }}-go-benchmark

.github/workflows/on-pull-request.yml

@@ -44,3 +44,69 @@ jobs:
 
       - name: Test
         run: go test -v -race -p=1 -count=1
+  go-bench:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # to be able to retrieve the last commit in master branch
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: './go.mod'
+          cache-dependency-path: './go.sum'
+          check-latest: true
+
+      - name: Run benchmark and store the output to a file
+        run: |
+          set -o pipefail
+          make bench | tee ${{ github.sha }}_bench_output.txt
+
+      - name: Get Master branch SHA
+        id: get-master-branch-sha
+        run: |
+          SHA=$(git rev-parse origin/master)
+          echo "sha=$SHA" >> $GITHUB_OUTPUT
+
+      - name: Get benchmark JSON from Master branch
+        id: cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ./cache/benchmark-data.json
+          key: ${{ steps.get-master-branch-sha.outputs.sha }}-${{ runner.os }}-go-benchmark
+
+      - name: Compare benchmarks with master
+        uses: benchmark-action/github-action-benchmark@v1
+        if: steps.cache.outputs.cache-hit == 'true'
+        with:
+          alert-threshold: "200%"
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: ${{ github.sha }}_bench_output.txt
+          # Where the benchmarks in master are (to compare)
+          external-data-json-path: ./cache/benchmark-data.json
+          # Do not save the data
+          save-data-file: false
+          # Workflow will fail when an alert happens
+          fail-on-alert: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Enable Job Summary for PRs
+          summary-always: true
+
+      - name: Run benchmarks but don't compare to master branch
+        uses: benchmark-action/github-action-benchmark@v1
+        if: steps.cache.outputs.cache-hit != 'true'
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: ${{ github.sha }}_bench_output.txt
+          # Write benchmarks to this file, do not publish to GitHub Pages
+          save-data-file: false
+          external-data-json-path: ./cache/benchmark-data.json
+          # Enable Job Summary for PRs
+          summary-always: true

Makefile

@@ -18,6 +18,10 @@ test:
 		go tool cover -html coverage.out -o coverage.html; \
 		exit $$ret)
 
+.PHONY: bench
+bench:
+	go test ./... -bench . -benchtime 5s -timeout 0 -run=XXX -benchmem
+
 .PHONY: docker
 docker:
 	docker build --build-arg VERSION=$(VERSION) -t ghcr.io/mailgun/gubernator:$(VERSION) .