Merge pull request #822 from cloudflare/benchmarks

Add benchmark-action/github-action-benchmark

.github/spellcheck/wordlist.txt

@@ -36,6 +36,7 @@ promql
 PromQL
 PRs
 prymitive
+samber
 SNI
 symlink
 symlinked

.github/workflows/benchmark.yml

@@ -1,13 +1,13 @@
 name: Benchmark Go code
 
 on:
-  push:
-    branches:
-      - main
   pull_request:
     branches:
       - main
 
+permissions:
+  pull-requests: write
+
 jobs:
   test-go:
     runs-on: ubuntu-latest
@@ -22,11 +22,26 @@ jobs:
         with:
           go-version-file: go.mod
 
-      - name: Test
-        run: make benchmark
+      - name: Fetch test rules
+        run: make -C cmd/pint/bench fetch
 
-      - name: CPU
-        run: go tool pprof -top cpu.prof
+      - name: Benchmark PR branch
+        run: |
+          make benchmark | tee new.txt
 
-      - name: Memory
-        run: go tool pprof -top mem.prof
+      - name: Benchmark main branch
+        run: |
+          git fetch origin main
+          git reset --hard FETCH_HEAD
+          make benchmark | tee old.txt
+
+      - name: Diff benchmarks
+        run: |
+          git reset --hard ${GITHUB_SHA}
+          make benchmark-diff
+
+      - name: Report
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          filePath: benchstat.txt
+          comment_tag: benchstat

.gitignore

@@ -1,5 +1,5 @@
 /pint
 /.cover
-/memprofile.out
 /dist/
 /.vscode/
+/cmd/pint/bench/rules

Makefile

@@ -74,11 +74,18 @@ coverhtml: test
 .PHONY: benchmark
 benchmark:
 	go test \
-		-v \
 		-count=10 \
 		-run=none \
 		-bench=. \
 		-benchmem \
-		-cpuprofile cpu.prof \
-		-memprofile mem.prof \
 		./cmd/pint
+
+$(GOBIN)/benchstat: tools/benchstat/go.mod tools/benchstat/go.sum
+	go install -modfile=tools/benchstat/go.mod golang.org/x/perf/cmd/benchstat
+.PHONY: benchmark-diff
+benchmark-diff: $(GOBIN)/benchstat
+	echo "Benchmark diff:" | tee benchstat.txt
+	echo "" | tee -a benchstat.txt
+	echo '```' | tee -a benchstat.txt
+	$(GOBIN)/benchstat old.txt new.txt | tee -a benchstat.txt
+	echo '```' | tee -a benchstat.txt

cmd/pint/bench/Makefile

@@ -0,0 +1,9 @@
+REVISION := 31a27fb9e0e778bd8fe6097aa58c8ea598fe9cec
+
+.PHONE: fetch
+fetch:
+	curl -sL -o archive.tar.gz https://github.com/samber/awesome-prometheus-alerts/archive/$(REVISION).tar.gz
+	tar -xf archive.tar.gz
+	rm -fr rules
+	mv awesome-prometheus-alerts-$(REVISION)/dist/rules rules
+	rm -fr awesome-prometheus-alerts-$(REVISION) archive.tar.gz

cmd/pint/bench/README.md

@@ -0,0 +1,7 @@
+# Benchmark data
+
+A collection of Prometheus rules copied from
+[samber/awesome-prometheus-alerts](https://github.com/samber/awesome-prometheus-alerts)
+and used to benchmark pint.
+
+Run `make fetch` first to download and unpack rules into `rules` folder.

cmd/pint/bench_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/pint/internal/config"
+	"github.com/cloudflare/pint/internal/discovery"
+	"github.com/cloudflare/pint/internal/git"
+	"github.com/cloudflare/pint/internal/log"
+)
+
+func BenchmarkFindEntries(b *testing.B) {
+	log.Setup(slog.LevelError, true)
+
+	finder := discovery.NewGlobFinder(
+		[]string{"bench/rules"},
+		git.NewPathFilter(nil, nil, nil),
+	)
+	for n := 0; n < b.N; n++ {
+		_, _ = finder.Find()
+	}
+}
+
+func BenchmarkCheckRules(b *testing.B) {
+	log.Setup(slog.LevelError, true)
+
+	finder := discovery.NewGlobFinder(
+		[]string{"bench/rules"},
+		git.NewPathFilter(nil, nil, nil),
+	)
+	entries, err := finder.Find()
+	if err != nil {
+		b.Errorf("Find() error: %s", err)
+		b.FailNow()
+	}
+
+	ctx := context.Background()
+	cfg, _ := config.Load("", false)
+	gen := config.NewPrometheusGenerator(cfg, prometheus.NewRegistry())
+	for n := 0; n < b.N; n++ {
+		_, _ = checkRules(ctx, 10, true, gen, cfg, entries)
+	}
+}

tools/benchstat/go.mod

@@ -0,0 +1,7 @@
+module _
+
+go 1.21.4
+
+require golang.org/x/perf v0.0.0-20231127181059-b53752263861
+
+require github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect

tools/benchstat/go.sum

@@ -0,0 +1,4 @@
+github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g=
+github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY=
+golang.org/x/perf v0.0.0-20231127181059-b53752263861 h1:oWUgm6RG0ZUKQeX7wxjnX+TFCrXDSheBtpiy5vpUdZg=
+golang.org/x/perf v0.0.0-20231127181059-b53752263861/go.mod h1:sKqUfjZtdM78wIFCYFnmu2FhhanvRUbp67Zgmj3TZPM=

tools/benchstat/main.go

@@ -0,0 +1,7 @@
+//go:build tools
+
+package tools
+
+import (
+	_ "golang.org/x/perf/cmd/benchstat"
+)