Perf/dma tail wide memset #416
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench verifier | |
| # Manual-only (/bench-verify, or workflow_dispatch to test workflow changes on a | |
| # branch — issue_comment always runs main's copy of this file, see profile-recursion.yml); | |
| # separate from /bench so they never share the bench server. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pairs: | |
| description: "ABBA pair count (2-40)" | |
| required: false | |
| default: "20" | |
| issue_comment: | |
| types: [created] | |
| concurrency: | |
| # Serialization is provided by the single self-hosted bench runner (jobs queue | |
| # for it). Never cancel a running bench: real /bench-verify comments share the | |
| # per-PR group and queue; any other comment (this workflow fires on every | |
| # issue_comment) gets a unique throwaway group so it can't sit in — or evict — | |
| # the real queue. | |
| group: ${{ startsWith(github.event.comment.body, '/bench-verify') && format('bench-verify-{0}', github.event.issue.number) || format('bench-verify-ignore-{0}', github.run_id) }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| verify: | |
| if: >- | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| startsWith(github.event.comment.body, '/bench-verify') && | |
| contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)) | |
| runs-on: [self-hosted, bench] | |
| # Job cap. On a cold runner the recursion BUILDS dominate: per ref a guest build, a | |
| # prover-test build, and a measuring-CLI build (the CLI is built FROM each ref so it | |
| # understands that ref's own guest syscalls). Cached in /tmp; build-std and the host | |
| # cargo target are both per ref (see the recursion step's env), so a ref's second and | |
| # later presets reuse its own compiled deps — but the two refs never share a dir. | |
| # 125 rather than 90 to absorb the continuation arms: a 20-tx continuation prove per | |
| # ref on each side of the verifier bench, plus a 20-tx (was 4-tx) block for the cycle | |
| # comparison. Keep this ABOVE the sum of the step caps below (50 + 70 = 120) so a | |
| # runaway step always trips its OWN timeout first: a step timeout still runs the | |
| # `always()` Post result step, whereas hitting the job cap is a cancellation and is far | |
| # less dependable about doing so — which would lose the comment entirely. | |
| timeout-minutes: 125 | |
| steps: | |
| - name: Acknowledge (react + occupancy notice) | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, content: 'eyes' | |
| }); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: '⏳ **Benchmark started** on the bench server. Two verifier arms (monolithic + continuations over an ethrex 20-tx block), then the recursion-guest cycle comparison, which adds guest builds on top — longer on a cold runner. The bench server is occupied until it finishes.' | |
| }); | |
| - name: Resolve PR head + pair count | |
| id: cfg | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUM: ${{ github.event.issue.number }} | |
| COMMENT_BODY: ${{ github.event.comment.body }} | |
| DISPATCH_PAIRS: ${{ github.event.inputs.pairs }} | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then | |
| # Testing this workflow's own changes: bench the dispatched branch vs main. | |
| HEAD_SHA="$GITHUB_SHA" | |
| N="${DISPATCH_PAIRS:-20}" | |
| else | |
| # Head SHA (not branch name) so fork PRs resolve and a mid-run force-push can't race. | |
| HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) | |
| # Optional pair count "/bench-verify 32"; default 20. | |
| N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-verify[[:space:]]*\([0-9]\+\).*|\1|p') | |
| N=${N:-20} | |
| fi | |
| echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" | |
| if ! [[ "$N" =~ ^[0-9]+$ ]]; then | |
| echo "::warning::pair count '$N' is not a number; using 20" | |
| N=20 | |
| fi | |
| if [ "$N" -lt 2 ] || [ "$N" -gt 40 ]; then | |
| echo "::warning::pair count $N out of range [2,40]; using 20" | |
| N=20 | |
| fi | |
| echo "pairs=$N" >> "$GITHUB_OUTPUT" | |
| - name: Checkout (full history for ref resolution) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Fetch PR head commit (works for fork PRs) | |
| if: github.event_name == 'issue_comment' | |
| env: | |
| PR_NUM: ${{ github.event.issue.number }} | |
| run: git fetch origin "pull/$PR_NUM/head" --quiet | |
| - name: Add cargo to PATH | |
| run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| - name: Run verifier benchmark | |
| id: run | |
| # Own cap so a hung continuation prove/verify can't eat the whole job budget and | |
| # starve the recursion step. On timeout `always()` still posts the failure tail. | |
| timeout-minutes: 50 | |
| env: | |
| HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} | |
| PAIRS: ${{ steps.cfg.outputs.pairs }} | |
| # The script defaults to the real block; this job pins the synthetic one. | |
| # /bench-verify reports TWO arms and the real path runs only the continuation | |
| # one (a real block does not fit monolithically), so inheriting the default | |
| # would silently drop an arm and break comparability with every verify number | |
| # recorded so far. | |
| WORKLOAD: synthetic | |
| run: | | |
| export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" | |
| set -o pipefail | |
| scripts/bench_verify.sh "$HEAD_SHA" origin/main "$PAIRS" 2>&1 | tee /tmp/verify_out.txt | |
| sed -n '/<!-- verify-abba-report -->/,$p' /tmp/verify_out.txt > /tmp/verify_result.txt | |
| # Additive: deterministic recursion-guest cycle+accelerator diff (PR vs main), in | |
| # two regimes: `min` (cheap canary over the empty diagnostic program) and | |
| # `blowup2-block` (the same verifier over a REAL ethrex 20-tx block proved with | |
| # continuations, via the `continuation` guest). One exact `execute --cycles` | |
| # reading per ref (no ABBA); blowup2-block's dumped blob is cached by ref SHA | |
| # (bench_recursion_cycles.sh), so a repeat run skips re-proving. | |
| # GUEST_TARGET_DIR and HOST_TARGET_DIR are BASE paths — bench_recursion_cycles.sh | |
| # appends the ref SHA so each worktree owns its target dirs. Sharing one dir across | |
| # refs made cargo mix rlibs from both worktrees: loudly on the guest side (broke | |
| # every regime after the first) and SILENTLY on the host side (cargo declared the | |
| # second ref fresh and the run measured the first ref's binary). | |
| # continue-on-error + `!cancelled()` isolate this from the verifier bench above. | |
| - name: Run recursion guest cycle benchmark | |
| id: recursion | |
| if: ${{ !cancelled() }} | |
| continue-on-error: true | |
| # Fail-fast under the job cap so a runaway build can't burn the whole job | |
| # (continue-on-error absorbs the timeout; the verifier verdict still posts). | |
| timeout-minutes: 70 | |
| env: | |
| HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} | |
| # Base paths for the per-ref guest and host target dirs (`<base>_<sha8>`), both | |
| # rooted under the script's /tmp cache dir: build-std and native deps survive | |
| # across runs (and across presets within a run) instead of being rebuilt cold. | |
| GUEST_TARGET_DIR: /tmp/recursion_cycles_run/guest_target | |
| HOST_TARGET_DIR: /tmp/recursion_cycles_run/host_target | |
| run: | | |
| export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" | |
| .github/scripts/run_recursion_bench.sh "$HEAD_SHA" | |
| - name: Post result | |
| if: always() | |
| uses: actions/github-script@v7 | |
| env: | |
| HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} | |
| PAIRS: ${{ steps.cfg.outputs.pairs }} | |
| OUTCOME: ${{ steps.run.outcome }} | |
| RECURSION_OUTCOME: ${{ steps.recursion.outcome }} | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const read = (p) => { try { return fs.readFileSync(p, 'utf8').trim(); } catch { return ''; } }; | |
| // Bound any raw-log fallback so a future header rename (empty *_result.txt) | |
| // can't dump the entire build log into the PR comment. | |
| const tail = (s, n) => s.split('\n').slice(-n).join('\n'); | |
| const head = (process.env.HEAD_SHA || '').slice(0, 10), pairs = process.env.PAIRS; | |
| let body = `## Verifier benchmark — \`${head}\` vs \`main\` (${pairs} pairs, monolithic + continuations)\n\n`; | |
| if (process.env.OUTCOME === 'success') { | |
| const res = read('/tmp/verify_result.txt') || tail(read('/tmp/verify_out.txt'), 30); | |
| body += res + '\n'; | |
| // Scope this to the rows it actually describes: only the Verify-time rows are | |
| // ABBA. It used to be a blanket claim, which was wrong for the proof sizes here | |
| // and for every guest-cycle number in the section below. | |
| body += '\n<sub>Verify-time rows only: drift-free interleaved A/B/B/A, with paired-t '; | |
| body += 'and exact Wilcoxon — trust the verdict when the two agree. Proof sizes are '; | |
| body += 'single exact readings (no averaging). - = PR faster.</sub>\n'; | |
| } else { | |
| // A step timeout or OOM kill takes the whole script down, so the graceful | |
| // CONT_SKIP path never runs. bench_verify.sh renders the monolithic report as | |
| // soon as that arm finishes, so post it rather than throwing away a verdict | |
| // that was already measured. Path is $WORK/result_mono.txt in that script. | |
| const mono = read('/tmp/verify_run/result_mono.txt'); | |
| if (mono) { | |
| body += '⚠️ Run did not complete — the monolithic arm had already finished, '; | |
| body += 'so its result is below. The continuation arm is missing.\n\n' + mono + '\n'; | |
| } | |
| body += `❌ Run failed. Last log lines:\n\n` + '```\n' + tail(read('/tmp/verify_out.txt'), 30) + '\n```\n'; | |
| } | |
| // Additive recursion-guest cycle section, kept clearly separated from the | |
| // verifier verdict above so a failure here can't change how the bench reads. | |
| body += '\n---\n\n## Recursion guest cycles — verifier running INSIDE the VM (main vs PR)\n\n'; | |
| if (process.env.RECURSION_OUTCOME === 'success') { | |
| const rec = read('/tmp/recursion_result.txt') || tail(read('/tmp/recursion_out.txt'), 20); | |
| if (rec) { | |
| body += rec + '\n'; | |
| } else { | |
| body += '_No recursion comparison output was captured._\n'; | |
| } | |
| } else { | |
| const rtail = tail(read('/tmp/recursion_out.txt'), 20); | |
| body += '⚠️ Recursion cycle bench did not complete (does not affect the verifier verdict above).'; | |
| body += rtail ? ' Last log lines:\n\n' + '```\n' + rtail + '\n```\n' : '\n'; | |
| } | |
| // workflow_dispatch has no PR to comment on; write to the job summary instead. | |
| if (context.eventName !== 'issue_comment') { | |
| await core.summary.addRaw(body).write(); | |
| return; | |
| } | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const existing = comments.find(c => | |
| c.user.type === 'Bot' && c.body.includes('Verifier benchmark —')); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| comment_id: existing.id, body | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, body | |
| }); | |
| } | |
| # continue-on-error above protects the posted verifier result, not the failure itself. | |
| - name: Fail if recursion cycle bench didn't complete | |
| if: always() && steps.recursion.outcome != 'success' | |
| run: | | |
| echo "::error::Recursion cycle bench step did not complete (outcome=${{ steps.recursion.outcome }}) — see its log and the posted result above." | |
| exit 1 |