Diff benchmarks

.github/workflows/benchmark.yml

@@ -25,8 +25,20 @@ jobs:
       - name: Test
         run: make benchmark
 
-      - name: CPU
-        run: go tool pprof -top cpu.prof
+      - name: Download previous benchmark data
+        uses: actions/cache@v3
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
 
-      - name: Memory
-        run: go tool pprof -top mem.prof
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          tool: go
+          output-file-path: benchmark.txt
+          external-data-json-path: ./cache/benchmark-data.json
+          comment-on-alert: true
+          alert-threshold: "100%"
+          summary-always: true
+          auto-push: false

.gitignore

@@ -1,5 +1,5 @@
 /pint
 /.cover
-/memprofile.out
 /dist/
 /.vscode/
+/cmd/pint/bench/rules

Makefile

@@ -71,14 +71,31 @@ cover: test
 coverhtml: test
 	go tool cover -html=$(COVER_PROFILE)
 
+cmd/pint/bench/rules:
+	$(MAKE) -C cmd/pint/bench fetch
+
 .PHONY: benchmark
-benchmark:
+benchmark: cmd/pint/bench/rules
 	go test \
-		-v \
 		-count=10 \
 		-run=none \
 		-bench=. \
 		-benchmem \
-		-cpuprofile cpu.prof \
-		-memprofile mem.prof \
-		./cmd/pint
+		./cmd/pint | tee benchmark.txt
+
+$(GOBIN)/benchstat: tools/benchstat/go.mod tools/benchstat/go.sum
+	go install -modfile=tools/benchstat/go.mod golang.org/x/perf/cmd/benchstat
+
+.PHONY: benchmark-diff
+benchmark-diff: $(GOBIN)/benchstat
+	git diff --exit-code
+	$(MAKE) benchmark
+	mv benchmark.txt new.txt
+
+	git fetch origin main
+	git reset --hard FETCH_HEAD
+
+	$(MAKE) benchmark
+	mv benchmark.txt old.txt
+
+	$(GOBIN)/benchstat old.txt new.txt

cmd/pint/bench/Makefile

@@ -0,0 +1,9 @@
+REVISION := 31a27fb9e0e778bd8fe6097aa58c8ea598fe9cec
+
+.PHONE: fetch
+fetch:
+	curl -sL -o archive.tar.gz https://github.com/samber/awesome-prometheus-alerts/archive/$(REVISION).tar.gz
+	tar -xf archive.tar.gz
+	rm -fr rules
+	mv awesome-prometheus-alerts-$(REVISION)/dist/rules rules
+	rm -fr awesome-prometheus-alerts-$(REVISION) archive.tar.gz

cmd/pint/bench/README.md

@@ -0,0 +1,7 @@
+# Benchmark data
+
+A collection of Prometheus rules copied from
+[samber/awesome-prometheus-alerts](https://github.com/samber/awesome-prometheus-alerts)
+and used to benchmark pint.
+
+Run `make fetch` first to download and unpack rules into `rules` folder.

cmd/pint/bench_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/pint/internal/config"
+	"github.com/cloudflare/pint/internal/discovery"
+	"github.com/cloudflare/pint/internal/git"
+	"github.com/cloudflare/pint/internal/log"
+)
+
+func BenchmarkFindEntries(b *testing.B) {
+	log.Setup(slog.LevelError, true)
+
+	finder := discovery.NewGlobFinder(
+		[]string{"bench/rules"},
+		git.NewPathFilter(nil, nil, nil),
+	)
+	for n := 0; n < b.N; n++ {
+		_, _ = finder.Find()
+	}
+}
+
+func BenchmarkCheckRules(b *testing.B) {
+	log.Setup(slog.LevelError, true)
+
+	finder := discovery.NewGlobFinder(
+		[]string{"bench/rules"},
+		git.NewPathFilter(nil, nil, nil),
+	)
+	entries, err := finder.Find()
+	if err != nil {
+		b.Errorf("Find() error: %s", err)
+		b.FailNow()
+	}
+
+	ctx := context.Background()
+	cfg, _ := config.Load("", false)
+	gen := config.NewPrometheusGenerator(cfg, prometheus.NewRegistry())
+	for n := 0; n < b.N; n++ {
+		_, _ = checkRules(ctx, 10, true, gen, cfg, entries)
+	}
+}

tools/benchstat/go.mod

@@ -0,0 +1,7 @@
+module _
+
+go 1.21.4
+
+require golang.org/x/perf v0.0.0-20231127181059-b53752263861
+
+require github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect

tools/benchstat/go.sum

@@ -0,0 +1,4 @@
+github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g=
+github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY=
+golang.org/x/perf v0.0.0-20231127181059-b53752263861 h1:oWUgm6RG0ZUKQeX7wxjnX+TFCrXDSheBtpiy5vpUdZg=
+golang.org/x/perf v0.0.0-20231127181059-b53752263861/go.mod h1:sKqUfjZtdM78wIFCYFnmu2FhhanvRUbp67Zgmj3TZPM=

tools/benchstat/main.go

@@ -0,0 +1,7 @@
+//go:build tools
+
+package tools
+
+import (
+	_ "golang.org/x/perf/cmd/benchstat"
+)