From 6b8daedbf433ad02dd841dbccf07bc3830c1830b Mon Sep 17 00:00:00 2001 From: Priya Sundaram Date: Tue, 8 Sep 2026 07:43:57 +0000 Subject: [PATCH] perf: async fan-out + progress reporting for hacktoberfest prep tracker The daily prep job spent ~16 minutes because it looked up the changed files of every open 'awaiting reviews' PR one request at a time. Those lookups are independent, so fire them concurrently through a single httpx2.AsyncClient bounded by a small semaphore, and likewise resolve the tracked-row PR states and the issue/PR counts concurrently. Report progress to stderr (flushed) so a human watching the Actions log can see the job is alive. Runtime drops from ~16 min of serial round-trips to well under a minute. --- scripts/hacktoberfest_prep_update.py | 207 +++++++++++++++++++-------- 1 file changed, 147 insertions(+), 60 deletions(-) diff --git a/scripts/hacktoberfest_prep_update.py b/scripts/hacktoberfest_prep_update.py index deaa67db0341..6c6eb1de0fbe 100644 --- a/scripts/hacktoberfest_prep_update.py +++ b/scripts/hacktoberfest_prep_update.py @@ -15,10 +15,19 @@ 3. Exits non-zero once Hacktoberfest 2026 has begun (on or after 2026-10-01, UTC), so the prep window closing is loud rather than silent. -It only uses the standard library and the ``GITHUB_TOKEN`` provided by the -Actions runner, so there is nothing to install. +The slow part of the job is looking at the files touched by every open +``awaiting reviews`` PR (potentially a few hundred of them). Those requests +have no dependencies on one another, so they are fired concurrently through a +single ``httpx2.AsyncClient`` with a bounded concurrency limit — this turns a +long series of round-trips into a handful of batches and cuts the runtime from +~16 minutes to well under a minute. Progress is reported to stderr as it goes +so a human watching the Actions log can see it is alive. + +It only needs ``httpx2`` (the repo-standard HTTP client) and the +``GITHUB_TOKEN`` provided by the Actions runner. """ +import asyncio import datetime as dt import os import re @@ -34,6 +43,11 @@ AWAITING_LABEL = "awaiting reviews" HACKTOBERFEST_START = dt.date(2026, 10, 1) +# How many API requests to keep in flight at once. GitHub's authenticated +# primary limit is 5000/hour, but bursts of concurrent requests can trip the +# secondary limits, so keep this modest. +CONCURRENCY = 8 + # A tracked row looks like: ``12. [ ] #15144 awaiting reviews`` ROW_RE = re.compile( r"^(?P\d+)\.\s+\[(?P[ x])\]\s+#(?P\d+)\b(?P.*)$" @@ -41,8 +55,12 @@ STATS_HEADER = "## Automated statistics" -def _request(url: str, params: dict | None = None) -> tuple[dict | list, dict]: - """GET ``url`` and return ``(json_body, headers)``, retrying on 403/rate limit.""" +def _log(message: str) -> None: + """Emit a progress line to stderr, flushed so Actions shows it live.""" + print(message, file=sys.stderr, flush=True) + + +def _headers() -> dict[str, str]: headers = { "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", @@ -50,78 +68,133 @@ def _request(url: str, params: dict | None = None) -> tuple[dict | list, dict]: } if TOKEN: headers["Authorization"] = f"Bearer {TOKEN}" - for attempt in range(4): - resp = httpx2.get(url, params=params, headers=headers, timeout=30) - if resp.is_success: - return resp.json(), dict(resp.headers) - remaining = resp.headers.get("X-RateLimit-Remaining") - if resp.status_code in (403, 429) and remaining == "0": - reset = int(resp.headers.get("X-RateLimit-Reset", "0")) - wait = max(1, reset - int(time.time())) + 1 - print(f"Rate limited; sleeping {wait}s", file=sys.stderr) - time.sleep(min(wait, 90)) - continue - if resp.status_code >= 500 and attempt < 3: - time.sleep(2 * (attempt + 1)) - continue - resp.raise_for_status() - msg = f"giving up on {url}" - raise RuntimeError(msg) + return headers -def _search_count(query: str) -> int: - body, _ = _request(f"{API}/search/issues", {"q": query, "per_page": 1}) +async def _request( + client: httpx2.AsyncClient, + sem: asyncio.Semaphore, + url: str, + params: dict | None = None, +) -> tuple[dict | list, dict]: + """GET ``url`` and return ``(json_body, headers)``, retrying on 403/rate limit. + + The semaphore bounds how many of these run concurrently. + """ + async with sem: + for attempt in range(4): + resp = await client.get(url, params=params) + if resp.is_success: + return resp.json(), dict(resp.headers) + remaining = resp.headers.get("X-RateLimit-Remaining") + if resp.status_code in (403, 429) and remaining == "0": + reset = int(resp.headers.get("X-RateLimit-Reset", "0")) + wait = max(1, reset - int(time.time())) + 1 + _log(f"Rate limited on {url}; sleeping {min(wait, 90)}s") + await asyncio.sleep(min(wait, 90)) + continue + if resp.status_code >= 500 and attempt < 3: + await asyncio.sleep(2 * (attempt + 1)) + continue + resp.raise_for_status() + msg = f"giving up on {url}" + raise RuntimeError(msg) + + +async def _search_count( + client: httpx2.AsyncClient, sem: asyncio.Semaphore, query: str +) -> int: + body, _ = await _request( + client, sem, f"{API}/search/issues", {"q": query, "per_page": 1} + ) return int(body.get("total_count", 0)) # type: ignore[union-attr] -def pr_state(number: int) -> str | None: +async def pr_state( + client: httpx2.AsyncClient, sem: asyncio.Semaphore, number: int +) -> str | None: """Return ``"merged"`` / ``"closed"`` for a resolved PR, else ``None``.""" - body, _ = _request(f"{API}/repos/{REPO}/pulls/{number}") + body, _ = await _request(client, sem, f"{API}/repos/{REPO}/pulls/{number}") if body.get("state") == "open": # type: ignore[union-attr] return None return "merged" if body.get("merged_at") else "closed" # type: ignore[union-attr] -def top_awaiting_directories( - limit: int = 3, max_prs: int = 400 +async def top_awaiting_directories( + client: httpx2.AsyncClient, + sem: asyncio.Semaphore, + limit: int = 3, + max_prs: int = 400, ) -> list[tuple[str, int]]: """Count open ``awaiting reviews`` PRs by the top-level directory they touch.""" query = f'repo:{REPO} is:pr is:open label:"{AWAITING_LABEL}"' - counts: dict[str, int] = {} + numbers: list[int] = [] page = 1 - scanned = 0 - while scanned < max_prs: - body, _ = _request( + while len(numbers) < max_prs: + body, _ = await _request( + client, + sem, f"{API}/search/issues", {"q": query, "per_page": 100, "page": page}, ) items = body.get("items", []) # type: ignore[union-attr] if not items: break - for item in items: - number = item["number"] - files, _ = _request( - f"{API}/repos/{REPO}/pulls/{number}/files", {"per_page": 100} - ) - dirs = set() - for changed in files: # type: ignore[union-attr] - parts = changed["filename"].split("/") - if len(parts) > 1 and not parts[0].startswith("."): - dirs.add(parts[0]) - for directory in dirs: - counts[directory] = counts.get(directory, 0) + 1 - scanned += 1 - if scanned >= max_prs: - break + numbers.extend(item["number"] for item in items) if len(items) < 100: break page += 1 + numbers = numbers[:max_prs] + + total = len(numbers) + _log(f"Scanning changed files for {total} '{AWAITING_LABEL}' PR(s)...") + done = 0 + + async def dirs_for(number: int) -> set[str]: + nonlocal done + files, _ = await _request( + client, sem, f"{API}/repos/{REPO}/pulls/{number}/files", {"per_page": 100} + ) + dirs = set() + for changed in files: # type: ignore[union-attr] + parts = changed["filename"].split("/") + if len(parts) > 1 and not parts[0].startswith("."): + dirs.add(parts[0]) + done += 1 + if done % 25 == 0 or done == total: + _log(f" ...scanned {done}/{total} PR(s)") + return dirs + + results = await asyncio.gather(*(dirs_for(n) for n in numbers)) + + counts: dict[str, int] = {} + for dirs in results: + for directory in dirs: + counts[directory] = counts.get(directory, 0) + 1 ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) return ranked[:limit] -def refresh_checkboxes(lines: list[str]) -> tuple[list[str], int]: +async def refresh_checkboxes( + client: httpx2.AsyncClient, sem: asyncio.Semaphore, lines: list[str] +) -> tuple[list[str], int]: """Tick rows whose PR is now merged/closed. Returns (new_lines, n_updated).""" + # Collect the PR numbers of every still-unchecked tracked row, then look + # their states up concurrently. + pending = [ + int(match.group("pr")) + for line in lines + if (match := ROW_RE.match(line)) and match.group("mark") != "x" + ] + if pending: + _log(f"Checking {len(pending)} open tracker row(s) for resolution...") + states = dict( + zip( + pending, + await asyncio.gather(*(pr_state(client, sem, n) for n in pending)), + ) + ) + updated = 0 out: list[str] = [] for line in lines: @@ -129,7 +202,7 @@ def refresh_checkboxes(lines: list[str]) -> tuple[list[str], int]: if not match or match.group("mark") == "x": out.append(line) continue - state = pr_state(int(match.group("pr"))) + state = states.get(int(match.group("pr"))) if state is None: out.append(line) continue @@ -138,10 +211,14 @@ def refresh_checkboxes(lines: list[str]) -> tuple[list[str], int]: return out, updated -def build_stats_block() -> str: - open_issues = _search_count(f"repo:{REPO} is:issue is:open") - open_prs = _search_count(f"repo:{REPO} is:pr is:open") - awaiting = _search_count(f'repo:{REPO} is:pr is:open label:"{AWAITING_LABEL}"') +async def build_stats_block(client: httpx2.AsyncClient, sem: asyncio.Semaphore) -> str: + _log("Collecting open issue/PR counts...") + awaiting_query = f'repo:{REPO} is:pr is:open label:"{AWAITING_LABEL}"' + open_issues, open_prs, awaiting = await asyncio.gather( + _search_count(client, sem, f"repo:{REPO} is:issue is:open"), + _search_count(client, sem, f"repo:{REPO} is:pr is:open"), + _search_count(client, sem, awaiting_query), + ) today = dt.datetime.now(dt.UTC).date().isoformat() lines = [ @@ -162,7 +239,7 @@ def build_stats_block() -> str: ), "", ] - if top_dirs := top_awaiting_directories(): + if top_dirs := await top_awaiting_directories(client, sem): for rank, (directory, count) in enumerate(top_dirs, start=1): plural = "PR" if count == 1 else "PRs" lines.append(f"{rank}. `{directory}/` — {count} awaiting-reviews {plural}") @@ -178,29 +255,39 @@ def splice_stats(text: str, stats_block: str) -> str: return f"{head}\n\n{stats_block}\n" +async def compute(lines: list[str]) -> tuple[list[str], int, str]: + """Do all the network work: resolve tracker rows and build the stats block.""" + async with httpx2.AsyncClient(headers=_headers(), timeout=30) as client: + sem = asyncio.Semaphore(CONCURRENCY) + lines, n_updated = await refresh_checkboxes(client, sem, lines) + stats_block = await build_stats_block(client, sem) + return lines, n_updated, stats_block + + def main() -> int: + started = time.monotonic() with open(TRACKER, encoding="utf-8") as handle: text = handle.read() body_before_stats = text.split(STATS_HEADER, 1)[0] lines = body_before_stats.splitlines() - lines, n_updated = refresh_checkboxes(lines) - body = "\n".join(lines) - stats_block = build_stats_block() + lines, n_updated, stats_block = asyncio.run(compute(lines)) + + body = "\n".join(lines) new_text = splice_stats(body, stats_block) with open(TRACKER, "w", encoding="utf-8") as handle: handle.write(new_text) - print(f"Checked off {n_updated} newly-resolved pull request(s).") + elapsed = time.monotonic() - started + print(f"Checked off {n_updated} newly-resolved pull request(s) in {elapsed:.1f}s.") today = dt.datetime.now(dt.UTC).date() if today >= HACKTOBERFEST_START: - print( + _log( f"Hacktoberfest 2026 has begun ({today} >= {HACKTOBERFEST_START}); " - "the prep window is over — failing on purpose so this job is retired.", - file=sys.stderr, + "the prep window is over — failing on purpose so this job is retired." ) return 1 return 0