Benchmark LTO impact #11
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| # Manual, exploratory, on every runner OS the repo supports. Not part of | |
| # ci.yml: JMH runs are too slow to gate every push/PR. Two reports come out | |
| # of the same baseline (main) build: | |
| # | |
| # - Contestants: zstd-java (byte[] and MemorySegment) vs zstd-jni vs | |
| # aircompressor on identical input, plus reused-vs-fresh-per-call native | |
| # context overhead - the "is this library actually fast" evidence. | |
| # - LTO impact: baseline (main) vs an LTO variant branch (see | |
| # scripts/build-zstd.sh - currently -flto is Linux-only, zig's Mach-O | |
| # backend doesn't support it), so the real-vs-no-op split shows up as CI | |
| # evidence per platform instead of a manual claim. | |
| # | |
| # Results are consumable two ways: every table is both printed to the job log | |
| # (no sign-in needed, unlike the Step Summary UI) and uploaded as a | |
| # per-classifier artifact - `gh run download <run-id>` gets the raw JMH JSON | |
| # plus the rendered tables for offline diffing or archiving across runs. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| lto_ref: | |
| description: Branch/ref with the LTO change to compare against main. | |
| required: false | |
| default: experiment/lto-linux | |
| jobs: | |
| benchmark: | |
| name: ${{ matrix.classifier }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - { os: ubuntu-latest, classifier: linux-x86_64 } | |
| - { os: ubuntu-24.04-arm, classifier: linux-aarch64 } | |
| - { os: macos-14, classifier: osx-aarch64 } | |
| - { os: windows-latest, classifier: windows-x86_64 } | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout baseline (main, with zstd submodule) | |
| uses: actions/checkout@v7 | |
| with: | |
| submodules: recursive | |
| path: baseline | |
| - name: Checkout LTO variant (${{ inputs.lto_ref }}, with zstd submodule) | |
| uses: actions/checkout@v7 | |
| with: | |
| submodules: recursive | |
| ref: ${{ inputs.lto_ref }} | |
| path: lto | |
| - name: Set up JDK 25 | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: temurin | |
| java-version: '25' | |
| cache: maven | |
| - name: Set up Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 | |
| with: | |
| version: 0.16.0 | |
| # Anchored on the package-separator dot so `CompressBenchmark` doesn't | |
| # also match `MultiThreadCompressBenchmark` (JMH's filter is a | |
| # substring/regex search) - that class has no zstdJni/aircompressor | |
| # peers, so it only pollutes the contestants table with empty cells. | |
| # GoldenCorpusBenchmark stays excluded - keeps this to the quick | |
| # synthetic-payload suite, not the slower real-corpus one. | |
| - name: Build + run baseline benchmark | |
| shell: bash | |
| working-directory: baseline | |
| run: | | |
| ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests | |
| java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \ | |
| -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../baseline-results.json | |
| - name: Build + run LTO-variant benchmark | |
| shell: bash | |
| working-directory: lto | |
| run: | | |
| ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests | |
| java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \ | |
| -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../lto-results.json | |
| - name: Report contestants (zstd-java vs zstd-jni vs aircompressor) | |
| if: always() | |
| shell: bash | |
| run: | | |
| { | |
| echo "### ${{ matrix.classifier }}" | |
| echo | |
| python3 baseline/.github/scripts/format-contestants.py baseline-results.json | |
| } | tee contestants.md | tee -a "$GITHUB_STEP_SUMMARY" | |
| - name: Report LTO impact (baseline vs variant) | |
| if: always() | |
| shell: bash | |
| run: | | |
| { | |
| echo "### ${{ matrix.classifier }}" | |
| echo | |
| python3 baseline/.github/scripts/compare-benchmarks.py \ | |
| baseline-results.json lto-results.json | |
| echo | |
| } | tee comparison.md | tee -a "$GITHUB_STEP_SUMMARY" | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ matrix.classifier }} | |
| path: | | |
| baseline-results.json | |
| lto-results.json | |
| contestants.md | |
| comparison.md | |
| if-no-files-found: ignore | |
| retention-days: 30 |