Skip to content

perf: make Hacktoberfest prep tracker fast + verbose (async fan-out, progress) #15226

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cclauss merged 1 commit into from
Sep 8, 2026
Merged

perf: make Hacktoberfest prep tracker fast + verbose (async fan-out, progress) #15226

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 147 additions & 60 deletions scripts/hacktoberfest_prep_update.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,19 @@
3. Exits non-zero once Hacktoberfest 2026 has begun (on or after
2026-10-01, UTC), so the prep window closing is loud rather than silent.

It only uses the standard library and the ``GITHUB_TOKEN`` provided by the
Actions runner, so there is nothing to install.
The slow part of the job is looking at the files touched by every open
``awaiting reviews`` PR (potentially a few hundred of them). Those requests
have no dependencies on one another, so they are fired concurrently through a
single ``httpx2.AsyncClient`` with a bounded concurrency limit — this turns a
long series of round-trips into a handful of batches and cuts the runtime from
~16 minutes to well under a minute. Progress is reported to stderr as it goes
so a human watching the Actions log can see it is alive.

It only needs ``httpx2`` (the repo-standard HTTP client) and the
``GITHUB_TOKEN`` provided by the Actions runner.
"""

import asyncio
import datetime as dt
import os
import re
Expand All @@ -34,102 +43,166 @@
AWAITING_LABEL = "awaiting reviews"
HACKTOBERFEST_START = dt.date(2026, 10, 1)

# How many API requests to keep in flight at once. GitHub's authenticated
# primary limit is 5000/hour, but bursts of concurrent requests can trip the
# secondary limits, so keep this modest.
CONCURRENCY = 8

# A tracked row looks like: ``12. [ ] #15144 awaiting reviews``
ROW_RE = re.compile(
r"^(?P<idx>\d+)\.\s+\[(?P<mark>[ x])\]\s+#(?P<pr>\d+)\b(?P<rest>.*)$"
)
STATS_HEADER = "## Automated statistics"


def _request(url: str, params: dict | None = None) -> tuple[dict | list, dict]:
"""GET ``url`` and return ``(json_body, headers)``, retrying on 403/rate limit."""
def _log(message: str) -> None:
"""Emit a progress line to stderr, flushed so Actions shows it live."""
print(message, file=sys.stderr, flush=True)


def _headers() -> dict[str, str]:
headers = {
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "hacktoberfest-prep-bot",
}
if TOKEN:
headers["Authorization"] = f"Bearer {TOKEN}"
for attempt in range(4):
resp = httpx2.get(url, params=params, headers=headers, timeout=30)
if resp.is_success:
return resp.json(), dict(resp.headers)
remaining = resp.headers.get("X-RateLimit-Remaining")
if resp.status_code in (403, 429) and remaining == "0":
reset = int(resp.headers.get("X-RateLimit-Reset", "0"))
wait = max(1, reset - int(time.time())) + 1
print(f"Rate limited; sleeping {wait}s", file=sys.stderr)
time.sleep(min(wait, 90))
continue
if resp.status_code >= 500 and attempt < 3:
time.sleep(2 * (attempt + 1))
continue
resp.raise_for_status()
msg = f"giving up on {url}"
raise RuntimeError(msg)
return headers


def _search_count(query: str) -> int:
body, _ = _request(f"{API}/search/issues", {"q": query, "per_page": 1})
async def _request(
client: httpx2.AsyncClient,
sem: asyncio.Semaphore,
url: str,
params: dict | None = None,
) -> tuple[dict | list, dict]:
"""GET ``url`` and return ``(json_body, headers)``, retrying on 403/rate limit.

The semaphore bounds how many of these run concurrently.
"""
async with sem:
for attempt in range(4):
resp = await client.get(url, params=params)
if resp.is_success:
return resp.json(), dict(resp.headers)
remaining = resp.headers.get("X-RateLimit-Remaining")
if resp.status_code in (403, 429) and remaining == "0":
reset = int(resp.headers.get("X-RateLimit-Reset", "0"))
wait = max(1, reset - int(time.time())) + 1
_log(f"Rate limited on {url}; sleeping {min(wait, 90)}s")
await asyncio.sleep(min(wait, 90))
continue
if resp.status_code >= 500 and attempt < 3:
await asyncio.sleep(2 * (attempt + 1))
continue
resp.raise_for_status()
msg = f"giving up on {url}"
raise RuntimeError(msg)


async def _search_count(
client: httpx2.AsyncClient, sem: asyncio.Semaphore, query: str
) -> int:
body, _ = await _request(
client, sem, f"{API}/search/issues", {"q": query, "per_page": 1}
)
return int(body.get("total_count", 0)) # type: ignore[union-attr]


def pr_state(number: int) -> str | None:
async def pr_state(
client: httpx2.AsyncClient, sem: asyncio.Semaphore, number: int
) -> str | None:
"""Return ``"merged"`` / ``"closed"`` for a resolved PR, else ``None``."""
body, _ = _request(f"{API}/repos/{REPO}/pulls/{number}")
body, _ = await _request(client, sem, f"{API}/repos/{REPO}/pulls/{number}")
if body.get("state") == "open": # type: ignore[union-attr]
return None
return "merged" if body.get("merged_at") else "closed" # type: ignore[union-attr]


def top_awaiting_directories(
limit: int = 3, max_prs: int = 400
async def top_awaiting_directories(
client: httpx2.AsyncClient,
sem: asyncio.Semaphore,
limit: int = 3,
max_prs: int = 400,
) -> list[tuple[str, int]]:
"""Count open ``awaiting reviews`` PRs by the top-level directory they touch."""
query = f'repo:{REPO} is:pr is:open label:"{AWAITING_LABEL}"'
counts: dict[str, int] = {}
numbers: list[int] = []
page = 1
scanned = 0
while scanned < max_prs:
body, _ = _request(
while len(numbers) < max_prs:
body, _ = await _request(
client,
sem,
f"{API}/search/issues",
{"q": query, "per_page": 100, "page": page},
)
items = body.get("items", []) # type: ignore[union-attr]
if not items:
break
for item in items:
number = item["number"]
files, _ = _request(
f"{API}/repos/{REPO}/pulls/{number}/files", {"per_page": 100}
)
dirs = set()
for changed in files: # type: ignore[union-attr]
parts = changed["filename"].split("/")
if len(parts) > 1 and not parts[0].startswith("."):
dirs.add(parts[0])
for directory in dirs:
counts[directory] = counts.get(directory, 0) + 1
scanned += 1
if scanned >= max_prs:
break
numbers.extend(item["number"] for item in items)
if len(items) < 100:
break
page += 1
numbers = numbers[:max_prs]

total = len(numbers)
_log(f"Scanning changed files for {total} '{AWAITING_LABEL}' PR(s)...")
done = 0

async def dirs_for(number: int) -> set[str]:
nonlocal done
files, _ = await _request(
client, sem, f"{API}/repos/{REPO}/pulls/{number}/files", {"per_page": 100}
)
dirs = set()
for changed in files: # type: ignore[union-attr]
parts = changed["filename"].split("/")
if len(parts) > 1 and not parts[0].startswith("."):
dirs.add(parts[0])
done += 1
if done % 25 == 0 or done == total:
_log(f" ...scanned {done}/{total} PR(s)")
return dirs

results = await asyncio.gather(*(dirs_for(n) for n in numbers))

counts: dict[str, int] = {}
for dirs in results:
for directory in dirs:
counts[directory] = counts.get(directory, 0) + 1
ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
return ranked[:limit]


def refresh_checkboxes(lines: list[str]) -> tuple[list[str], int]:
async def refresh_checkboxes(
client: httpx2.AsyncClient, sem: asyncio.Semaphore, lines: list[str]
) -> tuple[list[str], int]:
"""Tick rows whose PR is now merged/closed. Returns (new_lines, n_updated)."""
# Collect the PR numbers of every still-unchecked tracked row, then look
# their states up concurrently.
pending = [
int(match.group("pr"))
for line in lines
if (match := ROW_RE.match(line)) and match.group("mark") != "x"
]
if pending:
_log(f"Checking {len(pending)} open tracker row(s) for resolution...")
states = dict(
zip(
pending,
await asyncio.gather(*(pr_state(client, sem, n) for n in pending)),
)
)

updated = 0
out: list[str] = []
for line in lines:
match = ROW_RE.match(line)
if not match or match.group("mark") == "x":
out.append(line)
continue
state = pr_state(int(match.group("pr")))
state = states.get(int(match.group("pr")))
if state is None:
out.append(line)
continue
Expand All @@ -138,10 +211,14 @@ def refresh_checkboxes(lines: list[str]) -> tuple[list[str], int]:
return out, updated


def build_stats_block() -> str:
open_issues = _search_count(f"repo:{REPO} is:issue is:open")
open_prs = _search_count(f"repo:{REPO} is:pr is:open")
awaiting = _search_count(f'repo:{REPO} is:pr is:open label:"{AWAITING_LABEL}"')
async def build_stats_block(client: httpx2.AsyncClient, sem: asyncio.Semaphore) -> str:
_log("Collecting open issue/PR counts...")
awaiting_query = f'repo:{REPO} is:pr is:open label:"{AWAITING_LABEL}"'
open_issues, open_prs, awaiting = await asyncio.gather(
_search_count(client, sem, f"repo:{REPO} is:issue is:open"),
_search_count(client, sem, f"repo:{REPO} is:pr is:open"),
_search_count(client, sem, awaiting_query),
)
today = dt.datetime.now(dt.UTC).date().isoformat()

lines = [
Expand All @@ -162,7 +239,7 @@ def build_stats_block() -> str:
),
"",
]
if top_dirs := top_awaiting_directories():
if top_dirs := await top_awaiting_directories(client, sem):
for rank, (directory, count) in enumerate(top_dirs, start=1):
plural = "PR" if count == 1 else "PRs"
lines.append(f"{rank}. `{directory}/` — {count} awaiting-reviews {plural}")
Expand All @@ -178,29 +255,39 @@ def splice_stats(text: str, stats_block: str) -> str:
return f"{head}\n\n{stats_block}\n"


async def compute(lines: list[str]) -> tuple[list[str], int, str]:
"""Do all the network work: resolve tracker rows and build the stats block."""
async with httpx2.AsyncClient(headers=_headers(), timeout=30) as client:
sem = asyncio.Semaphore(CONCURRENCY)
lines, n_updated = await refresh_checkboxes(client, sem, lines)
stats_block = await build_stats_block(client, sem)
return lines, n_updated, stats_block


def main() -> int:
started = time.monotonic()
with open(TRACKER, encoding="utf-8") as handle:
text = handle.read()

body_before_stats = text.split(STATS_HEADER, 1)[0]
lines = body_before_stats.splitlines()
lines, n_updated = refresh_checkboxes(lines)
body = "\n".join(lines)

stats_block = build_stats_block()
lines, n_updated, stats_block = asyncio.run(compute(lines))

body = "\n".join(lines)
new_text = splice_stats(body, stats_block)

with open(TRACKER, "w", encoding="utf-8") as handle:
handle.write(new_text)

print(f"Checked off {n_updated} newly-resolved pull request(s).")
elapsed = time.monotonic() - started
print(f"Checked off {n_updated} newly-resolved pull request(s) in {elapsed:.1f}s.")

today = dt.datetime.now(dt.UTC).date()
if today >= HACKTOBERFEST_START:
print(
_log(
f"Hacktoberfest 2026 has begun ({today} >= {HACKTOBERFEST_START}); "
"the prep window is over — failing on purpose so this job is retired.",
file=sys.stderr,
"the prep window is over — failing on purpose so this job is retired."
)
return 1
return 0
Expand Down
Loading