diff --git a/scripts/sweep-lib.test.ts b/scripts/sweep-lib.test.ts new file mode 100644 index 0000000000..e9f2cf5ad3 --- /dev/null +++ b/scripts/sweep-lib.test.ts @@ -0,0 +1,183 @@ +import { test, expect } from "bun:test"; +import { + buildStaleQuery, + isStaleCandidate, + collectPages, + withRateLimitRetry, +} from "./sweep-lib.ts"; + +const CUTOFF = new Date("2026-06-25T13:45:00Z"); +const THRESHOLD = 10; + +// Helper: a minimal issue that IS a stale candidate; tests override one field +// at a time to prove each disqualifier works in isolation. +const candidate = (overrides: Record = {}) => ({ + number: 1, + updated_at: "2026-01-01T00:00:00Z", + locked: false, + assignees: [], + labels: [{ name: "bug" }], + reactions: { "+1": 0 }, + ...overrides, +}); + +// -- buildStaleQuery --------------------------------------------------------- + +test("query targets exactly the labelable set, with a date-only cutoff", () => { + expect(buildStaleQuery("anthropics", "claude-code", CUTOFF)).toBe( + "repo:anthropics/claude-code is:issue is:open updated:<2026-06-25 -label:stale -label:autoclose no:assignee" + ); +}); + +// -- isStaleCandidate -------------------------------------------------------- + +test("an old, unlabeled, unassigned, quiet issue is a candidate", () => { + expect(isStaleCandidate(candidate(), CUTOFF, THRESHOLD)).toBe(true); +}); + +test("pull requests are never candidates", () => { + expect( + isStaleCandidate(candidate({ pull_request: { url: "x" } }), CUTOFF, THRESHOLD) + ).toBe(false); +}); + +test("locked issues are not candidates", () => { + expect(isStaleCandidate(candidate({ locked: true }), CUTOFF, THRESHOLD)).toBe(false); +}); + +test("assigned issues are not candidates", () => { + expect( + isStaleCandidate(candidate({ assignees: [{ login: "a" }] }), CUTOFF, THRESHOLD) + ).toBe(false); +}); + +test("issues already labeled stale or autoclose are not candidates", () => { + expect( + isStaleCandidate(candidate({ labels: [{ name: "stale" }] }), CUTOFF, THRESHOLD) + ).toBe(false); + expect( + isStaleCandidate(candidate({ labels: [{ name: "autoclose" }] }), CUTOFF, THRESHOLD) + ).toBe(false); +}); + +test("issues at or above the upvote threshold are not candidates", () => { + expect( + isStaleCandidate(candidate({ reactions: { "+1": THRESHOLD } }), CUTOFF, THRESHOLD) + ).toBe(false); + expect( + isStaleCandidate(candidate({ reactions: { "+1": THRESHOLD - 1 } }), CUTOFF, THRESHOLD) + ).toBe(true); +}); + +test("issues updated on or after the cutoff are not candidates", () => { + expect( + isStaleCandidate(candidate({ updated_at: "2026-07-01T00:00:00Z" }), CUTOFF, THRESHOLD) + ).toBe(false); + expect( + isStaleCandidate(candidate({ updated_at: CUTOFF.toISOString() }), CUTOFF, THRESHOLD) + ).toBe(false); +}); + +test("missing optional fields do not disqualify an old issue", () => { + expect( + isStaleCandidate({ number: 2, updated_at: "2026-01-01T00:00:00Z" }, CUTOFF, THRESHOLD) + ).toBe(true); +}); + +// -- collectPages ------------------------------------------------------------ + +// Fake paginated backend: `pages` is the full dataset served 100 at a time. +const backend = (all: number[], perPage = 100) => { + const calls: number[] = []; + const fetchPage = async (page: number) => { + calls.push(page); + return all.slice((page - 1) * perPage, page * perPage); + }; + return { calls, fetchPage }; +}; + +test("collects every page until a short page, preserving order", async () => { + const all = Array.from({ length: 250 }, (_, i) => i); + const { calls, fetchPage } = backend(all); + const result = await collectPages(fetchPage); + expect(result).toEqual(all); + expect(calls).toEqual([1, 2, 3]); // 100, 100, 50 β€” stops on the short page +}); + +test("stops on an empty page when the last full page ends the dataset", async () => { + const all = Array.from({ length: 200 }, (_, i) => i); + const { calls, fetchPage } = backend(all); + const result = await collectPages(fetchPage); + expect(result).toEqual(all); + expect(calls).toEqual([1, 2, 3]); // page 3 is empty +}); + +test("a short first page needs exactly one fetch", async () => { + const { calls, fetchPage } = backend([1, 2, 3]); + const result = await collectPages(fetchPage); + expect(result).toEqual([1, 2, 3]); + expect(calls).toEqual([1]); +}); + +test("respects the maxPages safety cap", async () => { + const all = Array.from({ length: 500 }, (_, i) => i); + const { calls, fetchPage } = backend(all); + const result = await collectPages(fetchPage, { maxPages: 2 }); + expect(result).toEqual(all.slice(0, 200)); + expect(calls).toEqual([1, 2]); +}); + +// -- withRateLimitRetry -------------------------------------------------------- + +// Fake flaky call: fails `n` times with `error`, then returns "ok". +const failNTimes = (n: number, error: string) => { + let calls = 0; + const fn = async () => { + calls++; + if (calls <= n) throw new Error(error); + return "ok"; + }; + return { fn, calls: () => calls }; +}; + +test("returns the result without sleeping when the call succeeds", async () => { + const sleeps: number[] = []; + const { fn, calls } = failNTimes(0, ""); + const result = await withRateLimitRetry(fn, { + sleeper: async (ms) => void sleeps.push(ms), + }); + expect(result).toBe("ok"); + expect(calls()).toBe(1); + expect(sleeps).toEqual([]); +}); + +test("backs off and retries when GitHub reports a secondary rate limit", async () => { + const sleeps: number[] = []; + const { fn, calls } = failNTimes(2, "GitHub API 403: secondary rate limit"); + const result = await withRateLimitRetry(fn, { + sleeper: async (ms) => void sleeps.push(ms), + }); + expect(result).toBe("ok"); + expect(calls()).toBe(3); + expect(sleeps).toEqual([60000, 120000]); +}); + +test("retries 429 responses too", async () => { + const { fn, calls } = failNTimes(1, "GitHub API 429: too many requests"); + expect(await withRateLimitRetry(fn, { sleeper: async () => {} })).toBe("ok"); + expect(calls()).toBe(2); +}); + +test("rethrows non-rate-limit errors immediately", async () => { + const { fn, calls } = failNTimes(5, "GitHub API 500: boom"); + await expect(withRateLimitRetry(fn, { sleeper: async () => {} })).rejects.toThrow("500"); + expect(calls()).toBe(1); +}); + +test("gives up after the attempt budget", async () => { + const { fn, calls } = failNTimes(99, "GitHub API 403: rate limit"); + await expect( + withRateLimitRetry(fn, { attempts: 3, sleeper: async () => {} }) + ).rejects.toThrow("403"); + expect(calls()).toBe(3); +}); diff --git a/scripts/sweep-lib.ts b/scripts/sweep-lib.ts new file mode 100644 index 0000000000..645a955dcf --- /dev/null +++ b/scripts/sweep-lib.ts @@ -0,0 +1,80 @@ +// Pure decision logic for sweep.ts, split out so it can be unit tested +// without network access (same pattern as issue-lifecycle.ts). + +export interface SweepIssue { + number: number; + updated_at: string; + locked?: boolean; + pull_request?: unknown; + assignees?: unknown[]; + labels?: { name: string }[]; + reactions?: Record; +} + +// Search query for exactly the set markStale may label. The plain issues +// listing can't serve as a work queue: it interleaves PRs and permanently +// skipped issues (upvoted, assigned, already stale) which accumulate at the +// front of the oldest-updated-first sort until no candidate is reachable +// within any fixed page budget. The search API excludes all of those +// server-side. Search's `updated:` qualifier has date granularity, so the +// cutoff is truncated to a date; isStaleCandidate re-checks the exact time. +export function buildStaleQuery(owner: string, repo: string, cutoff: Date): string { + const cutoffDate = cutoff.toISOString().split("T")[0]; + return `repo:${owner}/${repo} is:issue is:open updated:<${cutoffDate} -label:stale -label:autoclose no:assignee`; +} + +// Client-side re-check of everything the query filters (the search index can +// lag) plus the two conditions search can't express: the exact cutoff time +// and the πŸ‘-reaction threshold (search's `reactions:` counts all kinds). +export function isStaleCandidate( + issue: SweepIssue, + cutoff: Date, + upvoteThreshold: number +): boolean { + if (issue.pull_request) return false; + if (issue.locked) return false; + if ((issue.assignees?.length ?? 0) > 0) return false; + if (issue.labels?.some((l) => l.name === "stale" || l.name === "autoclose")) return false; + if ((issue.reactions?.["+1"] ?? 0) >= upvoteThreshold) return false; + return new Date(issue.updated_at) < cutoff; +} + +// GitHub's secondary rate limits reject bursts with 403/429 and ask callers +// to pause before retrying. Back off attemptΓ—60s (the docs say "wait a few +// minutes") for those; anything else propagates immediately. +export async function withRateLimitRetry( + fn: () => Promise, + opts: { attempts?: number; sleeper?: (ms: number) => Promise } = {} +): Promise { + const attempts = opts.attempts ?? 3; + const sleeper = + opts.sleeper ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + for (let attempt = 1; ; attempt++) { + try { + return await fn(); + } catch (err) { + if (attempt >= attempts || !/GitHub API (403|429)/.test(String(err))) throw err; + console.log(`rate limited β€” waiting ${attempt}min before retrying`); + await sleeper(attempt * 60_000); + } + } +} + +// Snapshot a paginated listing in full BEFORE acting on it. Offset pagination +// over a listing that the loop itself is mutating (labeling reorders the +// updated-sort; closing removes items) skips one item per mutation at every +// page boundary β€” collecting first makes the walk immune to that. +export async function collectPages( + fetchPage: (page: number) => Promise, + opts: { perPage?: number; maxPages?: number } = {} +): Promise { + const perPage = opts.perPage ?? 100; + const maxPages = opts.maxPages ?? 30; + const results: T[] = []; + for (let page = 1; page <= maxPages; page++) { + const items = await fetchPage(page); + results.push(...items); + if (items.length < perPage) break; + } + return results; +} diff --git a/scripts/sweep.ts b/scripts/sweep.ts index 4709290be3..8f211acbd7 100644 --- a/scripts/sweep.ts +++ b/scripts/sweep.ts @@ -1,12 +1,24 @@ #!/usr/bin/env bun import { lifecycle, STALE_UPVOTE_THRESHOLD } from "./issue-lifecycle.ts"; +import { + buildStaleQuery, + collectPages, + isStaleCandidate, + withRateLimitRetry, + type SweepIssue, +} from "./sweep-lib.ts"; // -- const NEW_ISSUE = "https://github.com/anthropics/claude-code/issues/new/choose"; const DRY_RUN = process.argv.includes("--dry-run"); +// Bounds label writes per run while the backlog drains β€” same per-run cap and +// pacing as the lock-closed-issues workflow. +const MAX_STALE_LABELS_PER_RUN = 250; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + const CLOSE_MESSAGE = (reason: string) => `Closing for now β€” ${reason}. Please [open a new issue](${NEW_ISSUE}) if this is still relevant.`; @@ -51,36 +63,50 @@ async function markStale(owner: string, repo: string) { console.log(`\n=== marking stale (${staleDays}d inactive) ===`); - for (let page = 1; page <= 10; page++) { - const issues = await githubRequest( - `/repos/${owner}/${repo}/issues?state=open&sort=updated&direction=asc&per_page=100&page=${page}` + // The plain issues listing can't serve as a work queue here: it interleaves + // PRs and permanently skipped issues (upvoted, assigned, already stale), + // which pile up at the front of the oldest-updated-first sort until no + // candidate is reachable within any fixed page budget β€” and labeling + // reorders the listing underneath the pagination. Ask the search API for + // exactly the labelable set instead (same approach as lock-closed-issues). + const query = buildStaleQuery(owner, repo, cutoff); + const searchPage = async (page: number) => { + // Pace search requests: the search API's secondary rate limit rejects + // rapid bursts well before the 30-requests/minute primary limit. + if (page > 1) await sleep(2000); + const result = await withRateLimitRetry(() => + githubRequest<{ items?: (SweepIssue & { title?: string })[] }>( + `/search/issues?q=${encodeURIComponent(query)}&sort=updated&order=asc&per_page=100&page=${page}` + ) ); - if (issues.length === 0) break; - - for (const issue of issues) { - if (issue.pull_request) continue; - if (issue.locked) continue; - if (issue.assignees?.length > 0) continue; - - const updatedAt = new Date(issue.updated_at); - if (updatedAt > cutoff) return labeled; - - const alreadyStale = issue.labels?.some( - (l: any) => l.name === "stale" || l.name === "autoclose" - ); - if (alreadyStale) continue; - - const thumbsUp = issue.reactions?.["+1"] ?? 0; - if (thumbsUp >= STALE_UPVOTE_THRESHOLD) continue; + return result.items ?? []; + }; + const processed = new Set(); + + // Labeling removes an issue from the query results, so each round the + // window refills with older candidates. `processed` skips issues we chose + // not to label (upvote threshold) and ones the search index still returns. + // Rounds are needed because search serves at most 1000 results per query. + while (labeled < MAX_STALE_LABELS_PER_RUN) { + const issues = await collectPages(searchPage, { maxPages: 10 }); + const fresh = issues.filter((i) => !processed.has(i.number)); + if (fresh.length === 0) break; + + for (const issue of fresh) { + if (labeled >= MAX_STALE_LABELS_PER_RUN) break; + processed.add(issue.number); + if (!isStaleCandidate(issue, cutoff, STALE_UPVOTE_THRESHOLD)) continue; const base = `/repos/${owner}/${repo}/issues/${issue.number}`; if (DRY_RUN) { + const updatedAt = new Date(issue.updated_at); const age = Math.floor((Date.now() - updatedAt.getTime()) / 86400000); console.log(`#${issue.number}: would label stale (${age}d inactive) β€” ${issue.title}`); } else { await githubRequest(`${base}/labels`, "POST", { labels: ["stale"] }); console.log(`#${issue.number}: labeled stale β€” ${issue.title}`); + await sleep(1000); } labeled++; } @@ -97,56 +123,61 @@ async function closeExpired(owner: string, repo: string) { cutoff.setDate(cutoff.getDate() - days); console.log(`\n=== ${label} (${days}d timeout) ===`); - for (let page = 1; page <= 10; page++) { - const issues = await githubRequest( - `/repos/${owner}/${repo}/issues?state=open&labels=${label}&sort=updated&direction=asc&per_page=100&page=${page}` - ); - if (issues.length === 0) break; + // Snapshot the listing before closing anything: closing an issue removes + // it from this filtered listing, which shifts the offset pages underneath + // the walk and skips one issue per close at every page boundary. The + // stale listing alone also outgrew the old fixed cap of 10 pages. + const issues = await collectPages( + (page) => + githubRequest( + `/repos/${owner}/${repo}/issues?state=open&labels=${label}&sort=updated&direction=asc&per_page=100&page=${page}` + ).then((items) => (Array.isArray(items) ? items : [])), + { maxPages: 30 } + ); - for (const issue of issues) { - if (issue.pull_request) continue; - if (issue.locked) continue; + for (const issue of issues) { + if (issue.pull_request) continue; + if (issue.locked) continue; - const thumbsUp = issue.reactions?.["+1"] ?? 0; - if (thumbsUp >= STALE_UPVOTE_THRESHOLD) continue; + const thumbsUp = issue.reactions?.["+1"] ?? 0; + if (thumbsUp >= STALE_UPVOTE_THRESHOLD) continue; - const base = `/repos/${owner}/${repo}/issues/${issue.number}`; + const base = `/repos/${owner}/${repo}/issues/${issue.number}`; - const events = await githubRequest(`${base}/events?per_page=100`); + const events = await githubRequest(`${base}/events?per_page=100`); - const labeledAt = events - .filter((e) => e.event === "labeled" && e.label?.name === label) - .map((e) => new Date(e.created_at)) - .pop(); + const labeledAt = events + .filter((e) => e.event === "labeled" && e.label?.name === label) + .map((e) => new Date(e.created_at)) + .pop(); - if (!labeledAt || labeledAt > cutoff) continue; + if (!labeledAt || labeledAt > cutoff) continue; - // Skip if a non-bot user commented after the label was applied. - // The triage workflow should remove lifecycle labels on human - // activity, but check here too as a safety net. - const comments = await githubRequest( - `${base}/comments?since=${labeledAt.toISOString()}&per_page=100` - ); - const hasHumanComment = comments.some( - (c) => c.user && c.user.type !== "Bot" + // Skip if a non-bot user commented after the label was applied. + // The triage workflow should remove lifecycle labels on human + // activity, but check here too as a safety net. + const comments = await githubRequest( + `${base}/comments?since=${labeledAt.toISOString()}&per_page=100` + ); + const hasHumanComment = comments.some( + (c) => c.user && c.user.type !== "Bot" + ); + if (hasHumanComment) { + console.log( + `#${issue.number}: skipping (human activity after ${label} label)` ); - if (hasHumanComment) { - console.log( - `#${issue.number}: skipping (human activity after ${label} label)` - ); - continue; - } - - if (DRY_RUN) { - const age = Math.floor((Date.now() - labeledAt.getTime()) / 86400000); - console.log(`#${issue.number}: would close (${label}, ${age}d old) β€” ${issue.title}`); - } else { - await githubRequest(`${base}/comments`, "POST", { body: CLOSE_MESSAGE(reason) }); - await githubRequest(base, "PATCH", { state: "closed", state_reason: "not_planned" }); - console.log(`#${issue.number}: closed (${label})`); - } - closed++; + continue; + } + + if (DRY_RUN) { + const age = Math.floor((Date.now() - labeledAt.getTime()) / 86400000); + console.log(`#${issue.number}: would close (${label}, ${age}d old) β€” ${issue.title}`); + } else { + await githubRequest(`${base}/comments`, "POST", { body: CLOSE_MESSAGE(reason) }); + await githubRequest(base, "PATCH", { state: "closed", state_reason: "not_planned" }); + console.log(`#${issue.number}: closed (${label})`); } + closed++; } }