diff --git a/.github/workflows/update-recall-floors.yml b/.github/workflows/update-recall-floors.yml deleted file mode 100644 index a911633ac..000000000 --- a/.github/workflows/update-recall-floors.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Update Recall Floors - -# Runs after every successful stable Publish (release or workflow_dispatch retry). -# Dev builds (push → Publish) are excluded via the job condition. -# -# Downloads the benchmark-result.json artifact produced by the -# pre-publish-benchmark gate, then calls scripts/update-recall-floors.mjs to -# raise any language floor that improved. Opens a PR only when at least one -# floor changed. - -on: - workflow_run: - workflows: [Publish] - types: [completed] - -jobs: - update-floors: - name: Bump recall floors - # Only stable releases: skip dev builds (triggered by push) and failed runs. - if: > - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event != 'push' - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/setup-node@v6 - with: - node-version: "22" - - - name: Download benchmark results - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh run download ${{ github.event.workflow_run.id }} \ - --repo "$GITHUB_REPOSITORY" \ - -n benchmark-results-json - - - name: Update recall floors - # gh run download places the artifact contents in a subdirectory named - # after the artifact (benchmark-results-json/benchmark-result.json). - run: node scripts/update-recall-floors.mjs benchmark-results-json/benchmark-result.json - - - name: Check for changes - id: diff - run: | - if git diff --quiet; then - echo "changed=false" >> "$GITHUB_OUTPUT" - echo "No floors improved — skipping PR." - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - - - name: Open PR - if: steps.diff.outputs.changed == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - git fetch --tags - VERSION=$(git describe --tags --match "v*" --abbrev=0 2>/dev/null | sed 's/^v//' || echo "unknown") - BRANCH="chore/recall-floors-v${VERSION}" - - # Idempotent: if this branch already exists (e.g. re-run), reset it. - git checkout -B "$BRANCH" - git add tests/benchmarks/resolution/resolution-benchmark.test.ts - git commit -m "chore: bump recall floors after v${VERSION} release" - git push origin "$BRANCH" --force - - # Avoid duplicate PRs on re-runs. - if gh pr view "$BRANCH" --json state -q '.state' 2>/dev/null | grep -q OPEN; then - echo "PR for $BRANCH already open — skipping gh pr create." - else - RUN_URL="${{ github.event.workflow_run.html_url }}" - gh pr create \ - --base main \ - --head "$BRANCH" \ - --title "chore: bump recall floors after v${VERSION} release" \ - --body "Recall floors are a one-way ratchet: only floors that improved in this release are raised. Triggered automatically by a successful stable [Publish workflow run](${RUN_URL})." - fi diff --git a/scripts/update-recall-floors.mjs b/scripts/update-recall-floors.mjs deleted file mode 100644 index 5f7d43655..000000000 --- a/scripts/update-recall-floors.mjs +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env node -/** - * Updates per-language recall/precision floors in the resolution benchmark - * threshold table after a stable release. - * - * Reads benchmark-result.json (the merged artifact produced by the - * pre-publish-benchmark CI job, which stores per-language resolution metrics - * under the `resolution` key). For each language whose actual precision or - * recall exceeds the current floor, the floor is raised to - * floor(actual * 100) / 100 — rounded down to 2 decimal places so the gate - * stays slightly conservative. - * - * This is intentionally a one-way ratchet: floors are never lowered. - * - * Usage: node scripts/update-recall-floors.mjs - */ -import fs from 'node:fs'; -import path from 'node:path'; - -const resultsPath = process.argv[2]; -if (!resultsPath) { - console.error('Usage: update-recall-floors.mjs '); - process.exit(1); -} - -const THRESHOLD_FILE = path.join( - import.meta.dirname, - '../tests/benchmarks/resolution/resolution-benchmark.test.ts', -); - -const raw = JSON.parse(fs.readFileSync(resultsPath, 'utf-8')); -// benchmark-result.json stores per-language resolution metrics under `.resolution` -// (merged in CI by the "Merge resolution into build result" step). Accept both -// the nested form and a bare per-language map for local use. -const results = raw.resolution ?? raw; - -let content = fs.readFileSync(THRESHOLD_FILE, 'utf-8'); - -/** Format a floor value: keep at least one decimal place, no trailing zeros. */ -function fmt(n) { - const s = String(Math.floor(n * 100) / 100); - return s.includes('.') ? s : `${s}.0`; -} - -let updatedCount = 0; - -for (const [lang, metrics] of Object.entries(results)) { - if (typeof metrics?.precision !== 'number' || typeof metrics?.recall !== 'number') continue; - - // Match lines of the form: ` lang: { precision: X, recall: Y },` - const pattern = new RegExp( - `( ${lang}: \\{ precision: )([\\d.]+)(, recall: )([\\d.]+)( \\})`, - ); - const match = content.match(pattern); - if (!match) continue; - - const currentPrecision = parseFloat(match[2]); - const currentRecall = parseFloat(match[4]); - - const newPrecision = Math.max(currentPrecision, Math.floor(metrics.precision * 100) / 100); - const newRecall = Math.max(currentRecall, Math.floor(metrics.recall * 100) / 100); - - if (newPrecision === currentPrecision && newRecall === currentRecall) continue; - - content = content.replace(pattern, `$1${fmt(newPrecision)}$3${fmt(newRecall)}$5`); - console.log( - ` ${lang}: precision ${fmt(currentPrecision)} → ${fmt(newPrecision)}, recall ${fmt(currentRecall)} → ${fmt(newRecall)}`, - ); - updatedCount++; -} - -if (updatedCount > 0) { - fs.writeFileSync(THRESHOLD_FILE, content); - console.log(`\nBumped ${updatedCount} language threshold(s).`); -} else { - console.log('No thresholds improved — nothing to update.'); -}