diff --git a/.github/workflows/benchmark-pr.yaml b/.github/workflows/benchmark-pr.yaml index fb6728d..95de955 100644 --- a/.github/workflows/benchmark-pr.yaml +++ b/.github/workflows/benchmark-pr.yaml @@ -1,42 +1,9 @@ --- -name: Benchmarks on AMD64 +name: Benchmarks on PRs (AMD64) permissions: read-all on: [pull_request] jobs: - benchmark-pull-request: - runs-on: ubuntu-latest-8-cores - steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - with: - fetch-depth: 0 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - name: Run Benchmarks - run: | - BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare REF=${{ github.event.pull_request.base.sha }} - - run: | - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" - cat result.txt >> "$GITHUB_STEP_SUMMARY" - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" - cat <> "$GITHUB_STEP_SUMMARY" -
- The table shows the median and 90% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE of the pull request, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10). - The last row has the Geometric Mean (geomean) for the given rows in the table. - Refer to [benchstat's documentation](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) for more help. - EOL - - name: Validate results under acceptable limit - run: | - export MAX_ACCEPTABLE_DIFFERENCE=5 - while IFS= read -r line; do - # Get fourth value, which is the comparison with the base. - value="$(echo "$line" | awk '{print $4}')" - if [[ "$value" = +* ]] || [[ "$value" = -* ]]; then - if (( $(echo "${value//[^0-9.]/}"'>'"$MAX_ACCEPTABLE_DIFFERENCE" | bc -l) )); then - echo "::error::$value is above the maximum acceptable difference ($MAX_ACCEPTABLE_DIFFERENCE)" - exit 1 - fi - fi - done < <(grep geomean result.txt) + amd64: + uses: ./.github/workflows/benchmark-template.yaml + with: + benchGitRef: ${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/benchmark-releases.yaml b/.github/workflows/benchmark-releases.yaml new file mode 100644 index 0000000..6cc1c1f --- /dev/null +++ b/.github/workflows/benchmark-releases.yaml @@ -0,0 +1,13 @@ +--- +name: Nightly Benchmarks against last release (AMD64) +permissions: read-all +on: + schedule: + - cron: '10 5 * * *' # runs every day at 05:10 UTC + # workflow_dispatch enables manual testing of this job by maintainers + workflow_dispatch: +jobs: + amd64: + uses: ./.github/workflows/benchmark-template.yaml + with: + benchGitRef: release-1.3 diff --git a/.github/workflows/benchmark-template.yaml b/.github/workflows/benchmark-template.yaml new file mode 100644 index 0000000..08adf44 --- /dev/null +++ b/.github/workflows/benchmark-template.yaml @@ -0,0 +1,57 @@ +--- +name: Reusable Benchmark Template +on: + workflow_call: + inputs: + # which git reference to benchmark against + benchGitRef: + required: true + type: string + maxAcceptableDifferencePercent: + required: false + type: number + default: 5 + runs-on: + required: false + type: string + default: "['ubuntu-latest-8-cores']" +permissions: read-all + +jobs: + benchmark: + runs-on: ${{ fromJson(inputs.runs-on) }} + steps: + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + with: + fetch-depth: 0 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - name: Run Benchmarks + run: | + BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare REF=${{ inputs.benchGitRef }} + - run: | + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + cat result.txt >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + cat <> "$GITHUB_STEP_SUMMARY" +
+ The table shows the median and 90% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10). + The last row has the Geometric Mean (geomean) for the given rows in the table. + Refer to [benchstat's documentation](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) for more help. + EOL + - name: Validate results under acceptable limit + run: | + export MAX_ACCEPTABLE_DIFFERENCE=${{ inputs.maxAcceptableDifferencePercent }} + while IFS= read -r line; do + # Get fourth value, which is the comparison with the base. + value="$(echo "$line" | awk '{print $4}')" + if [[ "$value" = +* ]] || [[ "$value" = -* ]]; then + if (( $(echo "${value//[^0-9.]/}"'>'"$MAX_ACCEPTABLE_DIFFERENCE" | bc -l) )); then + echo "::error::$value is above the maximum acceptable difference ($MAX_ACCEPTABLE_DIFFERENCE)" + exit 1 + fi + fi + done < <(grep geomean result.txt)