chore: Split benchmarks

mrueg · mrueg · commit b62cd2fdec57 · 2025-09-01T13:47:04.000+02:00
Split benchmarks into two jobs, so we can run them more often.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -140,8 +140,37 @@ jobs:
 
     - name: Benchmark tests
       run: |
-        BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare
+        BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare-main
     - run: |
+        echo "Comparison against main branch" >> "$GITHUB_STEP_SUMMARY"
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat result.txt >> "$GITHUB_STEP_SUMMARY"
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat <<EOL >> "$GITHUB_STEP_SUMMARY"
+        <hr />
+        The table shows the median and 95% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10).
+        The last row has the Geometric Mean (geomean) for the given rows in the table.
+        Refer to <a href="https://pkg.go.dev/golang.org/x/perf/cmd/benchstat">benchstat's documentation</a> for more help.
+        EOL
+
+  ci-benchmark-tests-release:
+    name: ci-benchmark-tests-releasebranch
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
+      id: go
+
+    - name: Benchmark tests
+      run: |
+        BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare-release
+    - run: |
+        echo "Comparison against release branch" >> "$GITHUB_STEP_SUMMARY"
         echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
         cat result.txt >> "$GITHUB_STEP_SUMMARY"
         echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
diff --git a/Makefile b/Makefile
@@ -101,9 +101,15 @@ validate-template: generate-template
 # Runs benchmark tests on the current git ref and the last release and compares
 # the two.
 test-benchmark-compare:
-	@git fetch
-	./tests/compare_benchmarks.sh main 2
-	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH} 2
+	$(MAKE) test-benchmark-compare-main test-benchmark-compare-release
+	
+test-benchmark-compare-main:
+	@git fetch origin main
+	./tests/compare_benchmarks.sh main 6
+
+test-benchmark-compare-release:
+	@git fetch origin ${LATEST_RELEASE_BRANCH}
+	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH} 6
 
 all: all-container
 
