diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4182e6f..74cb1ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,13 @@ jobs: - name: Check content schema run: bun run check:content-schema working-directory: docs-site + # check:coverage recomputes the coverage dashboard's tables from the tree + # and fails if the page disagrees, so its numbers cannot go stale the way + # they did when they were hand-transcribed. Python only; no extra setup + # beyond the runner's python3, same as check:api-drift above. + - name: Check docs coverage + run: bun run check:coverage + working-directory: docs-site - name: Check documented Rust examples compile run: bun run check:examples working-directory: docs-site diff --git a/docs-site/coverage_allowlist.toml b/docs-site/coverage_allowlist.toml new file mode 100644 index 0000000..1c5fc52 --- /dev/null +++ b/docs-site/coverage_allowlist.toml @@ -0,0 +1,23 @@ +# Public modules with no documentation page yet. +# +# `scripts/docs/check_coverage.py` fails on any public module in +# `crates/openquant/src/lib.rs` or `python/openquant/` that has no page under +# `src/content/docs/modules/`. An entry here is the only way to keep such a gap +# in the tree, and it costs a reason and an expiry date — after which the gate +# fails again, so a gap cannot be parked here indefinitely. +# +# Delete the entry when you write the page; the gate fails on an exemption for a +# module that turns out to be documented, because a stale exemption is how an +# allowlist stops meaning anything. +# +# [rust.some_module] # or [python.some_module] +# reason = "why there is no page yet" +# expires = 2026-12-31 # bare date, not a string + +[rust.data_processing] +reason = "Internal preprocessing helpers; no stable public surface to document yet." +expires = 2026-12-31 + +[python.bars] +reason = "Needs a page — it is a whole stage of the Python Core Workflow. Tracked, not accepted." +expires = 2026-12-31 diff --git a/docs-site/package.json b/docs-site/package.json index 8078079..bfe0331 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -10,9 +10,10 @@ "check:links": "node scripts/check-links.mjs", "check:api-drift": "python3 ../scripts/generate_api_inventory.py --check", "check:content-schema": "node scripts/check-content-schema.mjs", + "check:coverage": "python3 ../scripts/docs/check_coverage.py --check", "check:examples": "node scripts/check-doc-examples.mjs", "check:python-examples": "node scripts/check-doc-python-examples.mjs", - "check:docs": "bun run build && bun run check:links && bun run check:api-drift && bun run check:content-schema && bun run check:examples && bun run check:python-examples" + "check:docs": "bun run build && bun run check:links && bun run check:api-drift && bun run check:content-schema && bun run check:coverage && bun run check:examples && bun run check:python-examples" }, "devDependencies": { "astro": "^5.5.0" diff --git a/docs-site/src/content/docs/coverage.md b/docs-site/src/content/docs/coverage.md index 9b29cc6..50993d9 100644 --- a/docs-site/src/content/docs/coverage.md +++ b/docs-site/src/content/docs/coverage.md @@ -1,7 +1,8 @@ --- title: Coverage Dashboard description: What is documented, what is not, and the commands that produce those numbers. -status: draft +status: reviewed +last_validated: '2026-08-30' audience: - quant-dev - platform-engineering @@ -14,70 +15,84 @@ command; if the answer differs, the page is out of date and the command is right. A coverage page maintained any other way tells you what someone hoped was true on the day they wrote it. -Numbers below were measured on **2026-08-30**. - -## Documentation status +That used to be an aspiration. The three tables below are now generated: ```bash -cd docs-site && node scripts/check-content-schema.mjs +# verify — this is the step CI runs +cd docs-site && bun run check:coverage +# regenerate after a real change (path relative to the repo root) +python3 scripts/docs/check_coverage.py --write ``` -That gate runs in CI and prints a tally of every page by `status`. As of -the date above: +The gate recomputes each table from the tree and fails if this page disagrees +with it, so the numbers cannot quietly rot again. Everything outside the +generated regions is prose a human owns. + +Numbers below were regenerated on **2026-08-30**. + + +## Documentation status + +`status` is enforced by a second gate, `check:content-schema`, which also +refuses a review stamp that predates the page's last content change — so the +tally below cannot be bulk-applied. + + | `status` | Pages | Means | |---|---|---| | `generated` | 40 | Emitted from `src/data/moduleDocs.ts`. Nobody has read it. | -| `draft` | 8 | Hand-written, known incomplete. Claims nothing. | -| `reviewed` | 11 | A human read the page end to end. | -| `validated` | 0 | Reviewed *and* checked against the code. Nothing has earned this yet. | -| **Total** | **59** | | +| `draft` | 5 | Hand-written, known incomplete. Claims nothing. | +| `reviewed` | 12 | A human read the page end to end. | +| `validated` | 0 | Reviewed *and* checked against the code. | +| **Total** | **57** | | + The headline number is the last row of that table: **no page on this site is `validated`.** Until recently every page claimed `status: validated` with an identical `last_validated` date, because the value was a string literal in the page generator rather than a record of anyone's review. -The taxonomy in `docs-site/scripts/check-content-schema.mjs` now refuses -a stamp that predates the page's last content change, so the tally above -cannot be bulk-applied. ## Module documentation depth -```bash -cd docs-site -grep -c '^ module:' src/data/moduleDocs.ts # documented modules -grep -c '^ conceptOverview:' src/data/moduleDocs.ts # of those, the enriched tier -``` +Module pages are emitted from `src/data/moduleDocs.ts`. Every entry now fills +in `conceptOverview`, `whenToUse` and `relatedModules`, so every page carries +those sections; the split below is over the two optional ones, +`keyParameters` and `commonPitfalls`. + | | Count | |---|---| -| Modules with a documentation entry | 39 | -| …of which get the **enriched** template (`conceptOverview`, `whenToUse`, `keyParameters`, `commonPitfalls`, `relatedModules`) | 12 | -| …of which fall through to the **stub** template (`Subject` + one sentence per heading) | 27 | - -The stub tier is 69% of the module pages. Those pages have a heading -skeleton, one sentence under each heading, and a closing "Implementation -Notes" section that is a verbatim reprint of the page's own `risk_notes` -frontmatter. Depth on those pages is a generator problem, not a per-page -problem: adding `conceptOverview` to an entry in `moduleDocs.ts` moves it -to the enriched tier. +| Modules with a documentation page | 39 | +| …carrying the full template (**Key Parameters** and **Common Pitfalls** on top of the base sections) | 10 | +| …carrying the base template only | 29 | + + +A base-template page is not a stub — it has a concept overview, mathematical +foundations, usage examples, an API reference and risk notes. What it lacks is +the parameter table telling you which knobs matter and the pitfalls section +telling you how the module is usually misused. Both are the parts a reader +reaches for second, and both are a `moduleDocs.ts` edit rather than a per-page +rewrite. ## Code with no documentation entry at all -```bash -docd=$(grep -o 'module: "[^"]*"' docs-site/src/data/moduleDocs.ts | sed 's/module: "//;s/"//' | sort) -comm -13 <(echo "$docd") <(grep "^pub mod" crates/openquant/src/lib.rs | sed 's/pub mod //;s/;//' | sort) -comm -13 <(echo "$docd") <(ls python/openquant/*.py | xargs -n1 basename | sed 's/\.py//' | grep -v __init__ | sort) -``` +The gate derives this from `pub mod` in `crates/openquant/src/lib.rs` and the +modules in `python/openquant/`, against the `module:` frontmatter of the pages +under `src/content/docs/modules/`. A gap fails the gate unless +`docs-site/coverage_allowlist.toml` carries a reason and an unexpired date for +it. -| Surface | Undocumented | -|---|---| -| Rust (`crates/openquant/src/lib.rs`) | `data_processing` | -| Python (`python/openquant/`) | `bars` | + +| Surface | Undocumented | Exempt until | Why | +|---|---|---|---| +| Rust (`crates/openquant/src/lib.rs`) | `data_processing` | 2026-12-31 | Internal preprocessing helpers; no stable public surface to document yet. | +| Python (`python/openquant/`) | `bars` | 2026-12-31 | Needs a page — it is a whole stage of the Python Core Workflow. Tracked, not accepted. | + -`util` also appears in the raw diff, but it is a parent module whose two -children (`util::fast_ewma`, `util::volatility`) both have pages; it is -not a gap. +`util` also appears in the raw module list, but it is a parent namespace whose +two children (`util::fast_ewma`, `util::volatility`) both have pages; it is not +a gap, and the gate does not count it as one. `openquant.bars` is the sharper miss of the two. It has no module page even though [Python Core Workflow](/workflows/python-core-workflow/) @@ -91,23 +106,14 @@ Honestly: most of what you would want. - **Prose quality.** `status` records whether a human read a page, not whether the page is good. -- **Whether the examples run.** Nothing in CI executes the code blocks on - these pages. The commands on the setup pages were executed by hand; the - module pages' snippets were not, and several of them do not compile. +- **Whether the prose is right.** The gates count pages, sections and + modules. Nothing checks that a sentence describing an algorithm is true. - **API parity between Rust and Python.** `check:api-drift` (`scripts/generate_api_inventory.py --check`) tracks the *inventory* of public symbols, not whether both surfaces are documented equivalently. - That gate is **currently failing** on this branch. - -:::note[Why this page is still `draft`] -Three of the four tables above are hand-transcribed from commands run -once, on one day. That is better than the prose bullets this page used to -carry, and every figure can be re-derived in seconds — but it is still a -snapshot a human has to refresh, which is exactly the failure mode the -page warns about in its first paragraph. - -The fix is to emit this page from the gate that already computes the -tally. `check-content-schema.mjs` builds the status counts on every run -and throws them away after printing. Until it writes them out instead, -this page stays `draft`. -::: + +The examples are covered, though, and by execution rather than by counting: +`check:examples` compiles every documented Rust snippet against the real +crate and `check:python-examples` runs every documented Python block against +the built extension. Both are in the `docs-checks` CI job alongside this page's +gate. diff --git a/scripts/docs/check_coverage.py b/scripts/docs/check_coverage.py new file mode 100755 index 0000000..948fc95 --- /dev/null +++ b/scripts/docs/check_coverage.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Recompute the coverage dashboard's numbers, and fail when the page drifts. + +`docs-site/src/content/docs/coverage.md` opens by promising that every number on +it ships with the command that produced it. Until now the numbers were +hand-transcribed from commands someone ran once, which is the failure mode the +page itself warns about: by the time this script was written the page's status +tally was two pages out of date and its module-depth split was wrong by a +factor of three, because `moduleDocs.ts` was rewritten after the page was last +touched. + +So this is the command. It derives the three tables from the tree, and: + + --check (CI) compares them against the marked regions of the page and exits + non-zero on any drift. + --write rewrites those regions and re-stamps the measured-on date. + +Regions are delimited in the markdown by `` / +``. Everything outside them is prose that a human +owns; this script never touches it. + +Undocumented public modules are a hard failure unless `coverage_allowlist.toml` +carries a reason and an unexpired date for them — that file is the only place a +known gap is allowed to live, and it expires so gaps cannot be parked forever. +""" + +from __future__ import annotations + +import argparse +import re +import tomllib +from datetime import date +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DOCS_ROOT = ROOT / "docs-site" / "src" / "content" / "docs" +MODULE_PAGES = DOCS_ROOT / "modules" +COVERAGE_PAGE = DOCS_ROOT / "coverage.md" +LIB_RS = ROOT / "crates" / "openquant" / "src" / "lib.rs" +PY_SRC = ROOT / "python" / "openquant" +ALLOWLIST = ROOT / "docs-site" / "coverage_allowlist.toml" + +# Same taxonomy, same order, as docs-site/scripts/check-content-schema.mjs. +STATUS_ORDER = ("generated", "draft", "reviewed", "validated") +STATUS_MEANS = { + "generated": "Emitted from `src/data/moduleDocs.ts`. Nobody has read it.", + "draft": "Hand-written, known incomplete. Claims nothing.", + "reviewed": "A human read the page end to end.", + "validated": "Reviewed *and* checked against the code.", +} + +# The sections the module page generator emits for every module, versus the two +# it emits only for entries that fill in `keyParameters` / `commonPitfalls`. +BASE_SECTIONS = ("Concept Overview", "When to Use", "Related Modules") +FULL_SECTIONS = ("Key Parameters", "Common Pitfalls") + +PUB_MOD_RE = re.compile(r"^\s*pub\s+mod\s+([a-zA-Z0-9_]+)\s*;\s*$", re.MULTILINE) +FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---", re.DOTALL) +SCALAR_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*):\s*(.*)$") + + +def unquote(value: str) -> str: + v = value.strip() + if len(v) >= 2 and v[0] in "\"'" and v[-1] == v[0]: + return v[1:-1] + return v + + +def frontmatter(path: Path) -> dict[str, str]: + """Top-level scalar keys only — enough for `status` and `module`.""" + match = FRONTMATTER_RE.match(path.read_text(encoding="utf-8")) + if not match: + return {} + out: dict[str, str] = {} + for line in match.group(1).splitlines(): + if not line.strip() or line.startswith((" ", "\t", "#")): + continue + kv = SCALAR_RE.match(line) + if kv and kv.group(2).strip(): + out[kv.group(1)] = unquote(kv.group(2)) + return out + + +def doc_pages() -> list[Path]: + return sorted(p for p in DOCS_ROOT.rglob("*") if p.suffix in {".md", ".mdx"}) + + +def status_tally() -> dict[str, int]: + tally = dict.fromkeys(STATUS_ORDER, 0) + for page in doc_pages(): + status = frontmatter(page).get("status") + if status in tally: + tally[status] += 1 + return tally + + +def documented_modules() -> set[str]: + """Module names claimed by a module page, e.g. `labeling`, `util::volatility`.""" + names = set() + for page in sorted(MODULE_PAGES.glob("*.md")): + module = frontmatter(page).get("module") + if module: + names.add(module) + # A page for `util::volatility` documents part of `util`; the parent + # is a namespace, not a gap. + if "::" in module: + names.add(module.split("::", 1)[0]) + return names + + +def module_depth() -> tuple[int, int]: + """(pages carrying the full template, pages carrying only the base template).""" + full = base = 0 + for page in sorted(MODULE_PAGES.glob("*.md")): + if not frontmatter(page).get("module"): + continue # the index page + text = page.read_text(encoding="utf-8") + headings = {h for h in BASE_SECTIONS + FULL_SECTIONS if f"\n## {h}\n" in text} + if all(h in headings for h in FULL_SECTIONS): + full += 1 + else: + base += 1 + return full, base + + +def public_surfaces() -> dict[str, list[str]]: + rust = sorted(set(PUB_MOD_RE.findall(LIB_RS.read_text(encoding="utf-8")))) + python = sorted( + p.stem for p in PY_SRC.glob("*.py") if p.name != "__init__.py" + ) + return {"rust": rust, "python": python} + + +def load_allowlist() -> dict[str, dict[str, dict[str, str]]]: + if not ALLOWLIST.exists(): + return {} + with ALLOWLIST.open("rb") as fh: + return tomllib.load(fh) + + +def gaps(today: date) -> tuple[list[tuple[str, str, str, str]], list[str]]: + """Undocumented public modules, plus any problems with their exemptions. + + Returns the rows for the gap table — (surface, module, expires, reason) — + and a list of failures. A gap without a live exemption is a failure; so is + an exemption for a module that is in fact documented, because a stale + exemption is how an allowlist stops meaning anything. + """ + documented = documented_modules() + allow = load_allowlist() + rows: list[tuple[str, str, str, str]] = [] + failures: list[str] = [] + + labels = { + "rust": "Rust (`crates/openquant/src/lib.rs`)", + "python": "Python (`python/openquant/`)", + } + for surface, modules in public_surfaces().items(): + entries = allow.get(surface, {}) + for module in modules: + if module in documented: + if module in entries: + failures.append( + f"{surface}.{module} is exempted in " + f"{ALLOWLIST.relative_to(ROOT)} but is documented — drop the entry" + ) + continue + entry = entries.get(module) + if entry is None: + failures.append( + f"public {surface} module `{module}` has no module page and no " + f"entry in {ALLOWLIST.relative_to(ROOT)}" + ) + continue + reason = str(entry.get("reason", "")).strip() + expires = entry.get("expires") + if not reason: + failures.append(f"{surface}.{module}: allowlist entry needs a `reason`") + if not isinstance(expires, date): + failures.append( + f"{surface}.{module}: allowlist entry needs `expires` as a bare " + f"YYYY-MM-DD date (got {expires!r})" + ) + elif expires < today: + failures.append( + f"{surface}.{module}: exemption expired on {expires} — document " + f"the module or justify a new date" + ) + rows.append( + ( + labels[surface], + module, + expires.isoformat() if isinstance(expires, date) else str(expires), + reason or "—", + ) + ) + return rows, failures + + +def render_status_tally(tally: dict[str, int]) -> str: + lines = ["| `status` | Pages | Means |", "|---|---|---|"] + for status in STATUS_ORDER: + lines.append(f"| `{status}` | {tally[status]} | {STATUS_MEANS[status]} |") + lines.append(f"| **Total** | **{sum(tally.values())}** | |") + return "\n".join(lines) + + +def render_module_depth(full: int, base: int) -> str: + return "\n".join( + [ + "| | Count |", + "|---|---|", + f"| Modules with a documentation page | {full + base} |", + f"| …carrying the full template (**Key Parameters** and **Common Pitfalls** " + f"on top of the base sections) | {full} |", + f"| …carrying the base template only | {base} |", + ] + ) + + +def render_gaps(rows: list[tuple[str, str, str, str]]) -> str: + if not rows: + return "Every public module on both surfaces has a documentation page." + lines = ["| Surface | Undocumented | Exempt until | Why |", "|---|---|---|---|"] + for surface, module, expires, reason in rows: + lines.append(f"| {surface} | `{module}` | {expires} | {reason} |") + return "\n".join(lines) + + +def region_re(name: str) -> re.Pattern[str]: + return re.compile( + rf"(\n)(.*?)(\n)", + re.DOTALL, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--check", action="store_true", help="fail if the page has drifted (default)") + mode.add_argument("--write", action="store_true", help="rewrite the page's generated regions") + args = parser.parse_args() + + today = date.today() + tally = status_tally() + full, base = module_depth() + rows, failures = gaps(today) + + regions = { + "status-tally": render_status_tally(tally), + "module-depth": render_module_depth(full, base), + "gaps": render_gaps(rows), + "measured": f"Numbers below were regenerated on **{today.isoformat()}**.", + } + + page = COVERAGE_PAGE.read_text(encoding="utf-8") + drifted: list[str] = [] + for name, want in regions.items(): + pattern = region_re(name) + match = pattern.search(page) + if not match: + failures.append( + f"{COVERAGE_PAGE.relative_to(ROOT)} is missing the " + f"`` region" + ) + continue + if match.group(2) != want: + drifted.append(name) + # `--check` must not rewrite the file; building the new text is cheap + # and lets one code path serve both modes. + page = pattern.sub(lambda m, w=want: m.group(1) + w + m.group(3), page, count=1) + + # The measured-on date moves every day, so on --check it is informational: + # a gate that fails because a week went by teaches people to disable it. + drifted = [name for name in drifted if name != "measured"] + + if args.write: + if failures: + for failure in failures: + print(f"- {failure}") + print("\nRefusing to write: fix the coverage gaps above first.") + return 1 + COVERAGE_PAGE.write_text(page, encoding="utf-8") + print(f"wrote {COVERAGE_PAGE.relative_to(ROOT)}") + return 0 + + print( + f"docs pages: {sum(tally.values())} " + f"({', '.join(f'{tally[s]} {s}' for s in STATUS_ORDER if tally[s])})" + ) + print(f"module pages: {full + base} ({full} full template, {base} base template)") + print(f"undocumented public modules: {len(rows) or 'none'}") + + if drifted: + failures.append( + "coverage.md is stale in: " + + ", ".join(drifted) + + ". Run `python3 scripts/docs/check_coverage.py --write`." + ) + if failures: + print("\nCoverage check FAILED:") + for failure in failures: + print(f"- {failure}") + return 1 + print("\nCoverage check passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())