Improve benchmark running on github

.github/workflows/benchmarks-go.yml

@@ -0,0 +1,78 @@
+name: benchmarks-go
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+env:
+  GO_VERSION: 1.24.x
+  # Include sub-benchmarks under Native/STEF.
+  BENCH_REGEX: "Native/STEF($|/)"
+  BENCH_ITERATIONS: "10"
+  BENCH_COUNT: "10"
+
+jobs:
+  benchdiff:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout PR head
+        uses: actions/checkout@v4
+        with:
+          ref: refs/pull/${{ github.event.pull_request.number }}/head
+          fetch-depth: 0
+
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Install benchstat
+        run: |
+          echo "${HOME}/go/bin" >> "$GITHUB_PATH"
+          go install golang.org/x/perf/cmd/benchstat@latest
+
+      - name: Run benchmark diff
+        run: |
+          set -euo pipefail
+
+          base_sha="${{ github.event.pull_request.base.sha }}"
+          head_sha="${{ github.event.pull_request.head.sha }}"
+
+          rm -f bench_base.txt bench_current.txt benchstat.txt benchstat-comment.md
+
+          for i in $(seq 1 "$BENCH_ITERATIONS"); do
+            echo "Iteration ${i} of ${BENCH_ITERATIONS}"
+
+            echo "Checking out base commit ${base_sha}"
+            git checkout "$base_sha"
+            echo "Running benchmarks on base"
+            (cd benchmarks && go test -run noname -bench "$BENCH_REGEX" -benchmem ./... -count "$BENCH_COUNT") | tee -a bench_base.txt
+
+            echo "Checking out head commit ${head_sha}"
+            git checkout "$head_sha"
+            echo "Running benchmarks on PR head"
+            (cd benchmarks && go test -run noname -bench "$BENCH_REGEX" -benchmem ./... -count "$BENCH_COUNT") | tee -a bench_current.txt
+          done
+
+          benchstat bench_base.txt bench_current.txt | tee benchstat.txt
+
+          {
+            echo "## Go Benchstat report"
+            echo
+            echo "\`\`\`"
+            cat benchstat.txt
+            echo "\`\`\`"
+          } > benchstat-comment.md
+
+      - name: Post benchstat comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: benchstat
+          path: benchstat-comment.md

benchmarks/benchmarks_test.go

@@ -148,6 +148,7 @@ func BenchmarkSerializeNative(b *testing.B) {
 							log.Fatal(err)
 						}
 					}
+					b.StopTimer()
 					chart.RecordStacked(
 						b,
 						encoding.LongName(),

benchmarks/makefile

@@ -1,14 +1,5 @@
 GO=$(shell which go)
 
-BENCH_COUNT ?= 6
-CI_BENCHMARKS=STEF\|/STEF
-
-MASTER_BRANCH ?= main
-REF_NAME ?= $(shell git symbolic-ref -q --short HEAD || git describe --tags --exact-match)
-SHELL := /bin/bash
-VERSION=1.1
-BENCHSTAT=$(shell which benchstat)
-
 .PHONY: default
 default: build test
 
@@ -45,26 +36,3 @@ benchmark: gentestfiles
 build-ci:
 	make testcompat
 	make all
-
-.PHONY: benchmark-ci
-benchmark-ci: gentestfiles bench-run bench-stat-diff bench-stat
-
-.PHONY: bench-stat-cli
-bench-stat-cli:
-	@test -s $(GOPATH)/bin/benchstat || sudo GOFLAGS= GOBIN=$(GOPATH)/bin $(GO) install golang.org/x/perf/cmd/benchstat@latest
-
-.PHONY: bench-run
-bench-run:
-	@set -o pipefail && $(GO) test -test.benchmem -bench=$(CI_BENCHMARKS) -count=$(BENCH_COUNT) -run=^a  ./... | tee bench-$(REF_NAME).txt
-
-.PHONY: bench-stat-diff
-bench-stat-diff: bench-stat-cli
-	@test ! -e bench-$(MASTER_BRANCH).txt || benchstat bench-$(MASTER_BRANCH).txt bench-$(REF_NAME).txt
-
-.PHONY: bench-stat
-bench-stat: bench-stat-cli
-	benchstat bench-$(REF_NAME).txt
-
-.PHONY: update-charts
-update-charts:
-	UPDATE_BENCH_HTML=1 go test -run TestMetricsSize\|Multipart -bench \(alizeNative\|Pdata\) -benchtime=5s

benchmarks/scripts/benchdiff

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Save the current branch name
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+bench_count=${2:-1}
+base_branch=${3:-main}
+
+echo $base_branch
+
+# Ensure benchstat is installed
+if ! command -v benchstat &> /dev/null; then
+  echo "benchstat is not installed. Please install it first."
+  exit 1
+fi
+
+# Ensure we are not on the main branch
+if [ "$current_branch" == "main" ]; then
+  echo "You are already on the main branch. Switch to a different branch to compare."
+  exit 1
+fi
+
+# Get the benchmark name from the command-line argument, default to all benchmarks
+benchmark_name=${1:-.}
+
+rm bench_current.txt
+rm bench_base.txt
+
+for ((i=1; i<=$bench_count; i++)); do
+	echo "Iteration $i of $bench_count"
+
+	# Run benchmarks on the current branch
+	echo "Running benchmarks on branch: $current_branch"
+	go test -run noname -bench="$benchmark_name" -benchmem -benchtime 1s ./... -count 3 | tee -a bench_current.txt
+	if [ $? -ne 0 ]; then
+	  echo "Failed to run benchmarks on branch: $current_branch"
+	  exit 1
+	fi
+
+	# Checkout the main branch
+	echo "Switching to $base_branch branch..."
+	git checkout $base_branch
+	if [ $? -ne 0 ]; then
+	  echo "Failed to checkout $base_branch branch."
+	  exit 1
+	fi
+
+	# Run benchmarks on the main branch
+	echo "Running benchmarks on branch: $base_branch"
+	go test -run noname -bench="$benchmark_name" -benchmem -benchtime 1s ./... -count 3 | tee -a bench_base.txt
+	if [ $? -ne 0 ]; then
+	  echo "Failed to run benchmarks on branch: $base_branch"
+	  git checkout "$current_branch"
+	  exit 1
+	fi
+
+	# Restore the original branch
+	echo "Restoring branch: $current_branch"
+	git checkout "$current_branch"
+	if [ $? -ne 0 ]; then
+	  echo "Failed to restore branch: $current_branch"
+	  exit 1
+	fi
+
+done
+
+# Compare benchmark results using benchstat
+echo "Comparing benchmark results..."
+benchstat bench_base.txt bench_current.txt