fix: gate on hunk-level recall, fixed before Stage 3a

docs/hardening-plan.md

@@ -157,6 +157,18 @@ the API reviewer (recall 0.92); agent mode makes a silent reviewer cheap to
 run, and silence must not be able to flip `fail-on`. The cull table is
 unchanged: the floor applies to the gate only.
 
+**Amendment 2, 2026-10-23 — after runs 1–2, before Stage 3a.** The recall the
+gate reads becomes **hunk recall**: the share of in-scope defective hunks that
+draw at least one finding, under any rule. Reason: in run 2 the reviewer
+described two labelled defects correctly under rules other than the label
+(R4 / R7 / R12 / R13 overlap), so rule recall measures which name it picked
+as much as whether it saw. This is decided after looking at runs 1–2, and it
+changes neither outcome (hunk recall 1/15 and 3/15, both under 50%). It is
+fixed now so Stage 3a is scored against it, not fitted to it. Known
+weakness: a finding about a different problem in the same hunk still counts
+(1 of 3 in run 2); `unlabelled` is printed so a human can check. Rule recall
+stays in the output, ungated.
+
 ## Stage 3a — the file is the unit (~1 day)
 
 Feed the whole post-change file with the diff marked inside it. `file` and

scripts/corpus_cli.py

@@ -252,9 +252,9 @@ def prove(path: Path, out: Path | None) -> int:
         f"  stage3a if abstention > {STAGE3A_ABSTENTION_RATE:.0%}"
         f" and clean_noise < {STAGE3A_MAX_NOISE_RATE:.0%}",
         f"  fail-on ready if quiet_rate ≥ {FAIL_ON_MIN_QUIET_RATE:.0%},"
-        f" recall ≥ {FAIL_ON_MIN_RECALL:.0%}"
+        f" hunk recall ≥ {FAIL_ON_MIN_RECALL:.0%}"
         f" and no high rule > {FAIL_ON_MAX_HIGH_NOISE_RATE:.0%} clean noise"
-        " (recall floor amended 2026-10-23)",
+        " (recall floor amended 2026-10-23, hunk-level before Stage 3a)",
         f"  shape: ≥{MIN_CLEAN_HUNKS} clean, ≥{MIN_DEFECTIVE_HUNKS} defective",
         "",
     ]

src/lazycoder/corpus.py

@@ -203,6 +203,8 @@ class CorpusReport:
     caught: int
     missed: int
     per_rule: dict[RuleId, RuleVerdict] = field(default_factory=dict)
+    scored_defective_hunks: int = 0
+    flagged_defective_hunks: int = 0
 
     @property
     def quiet_rate(self) -> float:
@@ -218,9 +220,26 @@ def noise_per_clean_hunk(self) -> float:
 
     @property
     def recall(self) -> float:
+        """Rule recall: the labelled rule itself fired. Reported, not gated.
+
+        Rules overlap (R4 / R7 / R12 / R13 can all describe one bug), so this
+        partly measures which name the reviewer picked, not whether it saw.
+        """
         expected = self.caught + self.missed
         return self.caught / expected if expected else 1.0
 
+    @property
+    def hunk_recall(self) -> float:
+        """Share of in-scope defective hunks that drew at least one finding.
+
+        The gated recall since 2026-10-23 (pre-registered before Stage 3a).
+        It overstates detection when the finding is about something other
+        than the labelled defect; read `unlabelled` to see how often.
+        """
+        if not self.scored_defective_hunks:
+            return 1.0
+        return self.flagged_defective_hunks / self.scored_defective_hunks
+
 
 def report(results: list[HunkResult], rubric: ReviewRulesConfig) -> CorpusReport:
     clean = [r for r in results if r.label is Label.CLEAN]
@@ -259,6 +278,10 @@ def report(results: list[HunkResult], rubric: ReviewRulesConfig) -> CorpusReport
         caught=sum(caught.values()),
         missed=sum(missed.values()),
         per_rule=per_rule,
+        # A defective hunk whose labelled rules the preset never asks is out
+        # of scope here too, exactly as in rule recall.
+        scored_defective_hunks=sum(1 for r in defective if r.expected),
+        flagged_defective_hunks=sum(1 for r in defective if r.expected and r.fired),
     )
 
 
@@ -282,6 +305,9 @@ def report(results: list[HunkResult], rubric: ReviewRulesConfig) -> CorpusReport
 # Amended 2026-10-23, before any live corpus score exists: a reviewer that
 # never speaks has a perfect quiet rate. Agent mode makes that reviewer cheap
 # to run, so the gate also needs it to catch at least half the labelled defects.
+# Amended again 2026-10-23, after runs 1-2 and before Stage 3a: "catch" is
+# hunk-level (any rule fired on the defective hunk), not rule-level. Neither
+# past run changes outcome (hunk recall 1/15 and 3/15, both under the floor).
 FAIL_ON_MIN_RECALL = 0.50
 
 
@@ -350,20 +376,20 @@ class FailOnGate:
 
 def fail_on_gate(summary: CorpusReport, rubric: ReviewRulesConfig) -> FailOnGate:
     """Gate from the hardening plan: quiet_rate ≥ 80%, no high rule > 5% noise,
-    and recall ≥ 50% so silence alone cannot pass it."""
+    and hunk recall ≥ 50% so silence alone cannot pass it."""
     high_noisy = tuple(
         rule.id
         for rule in rubric.rules
         if rule.severity_if_unjustified is Severity.HIGH
         and summary.per_rule[rule.id].clean_noise_rate > FAIL_ON_MAX_HIGH_NOISE_RATE
     )
     quiet_ok = summary.quiet_rate >= FAIL_ON_MIN_QUIET_RATE
-    recall_ok = summary.recall >= FAIL_ON_MIN_RECALL
+    recall_ok = summary.hunk_recall >= FAIL_ON_MIN_RECALL
     ready = quiet_ok and recall_ok and not high_noisy
     if ready:
         reason = (
             f"quiet_rate {summary.quiet_rate:.0%} ≥ {FAIL_ON_MIN_QUIET_RATE:.0%}, "
-            f"recall {summary.recall:.0%} ≥ {FAIL_ON_MIN_RECALL:.0%} "
+            f"hunk recall {summary.hunk_recall:.0%} ≥ {FAIL_ON_MIN_RECALL:.0%} "
             f"and no high rule above {FAIL_ON_MAX_HIGH_NOISE_RATE:.0%} clean noise"
         )
     else:
@@ -373,7 +399,9 @@ def fail_on_gate(summary: CorpusReport, rubric: ReviewRulesConfig) -> FailOnGate
                 f"quiet_rate {summary.quiet_rate:.0%} < {FAIL_ON_MIN_QUIET_RATE:.0%}"
             )
         if not recall_ok:
-            parts.append(f"recall {summary.recall:.0%} < {FAIL_ON_MIN_RECALL:.0%}")
+            parts.append(
+                f"hunk recall {summary.hunk_recall:.0%} < {FAIL_ON_MIN_RECALL:.0%}"
+            )
         if high_noisy:
             parts.append(
                 "high-noise "
@@ -426,8 +454,12 @@ def format_corpus_summary(
         f"  ({summary.quiet_clean_hunks}/{summary.clean_hunks} clean hunks"
         " drew no comment)",
         f"noise       {summary.noise_per_clean_hunk:.2f} findings per clean hunk",
-        f"recall      {summary.recall:.0%}"
-        f"  ({summary.caught} caught, {summary.missed} missed)",
+        f"hunk recall {summary.hunk_recall:.0%}"
+        f"  ({summary.flagged_defective_hunks}/{summary.scored_defective_hunks}"
+        " defective hunks drew a finding; gated)",
+        f"rule recall {summary.recall:.0%}"
+        f"  ({summary.caught} caught, {summary.missed} missed; the labelled rule"
+        " itself fired)",
         f"\n{'rule':6}{'noise%':>8}{'caught':>8}{'missed':>8}{'abstain':>9}{'cull':>14}",
     ]
     plan = cull_plan(summary, rubric)

tests/test_canaries_corpus.py

@@ -333,6 +333,7 @@ def test_cli_corpus_scores_an_oracle_agent(
     assert code == 0
     assert "quiet rate  100%" in out
     # Only R7 defects are in scope for focused; R3/R4 ones are not "missed".
-    assert "recall      100%  (5 caught, 0 missed)" in out
+    assert "rule recall 100%  (5 caught, 0 missed" in out
+    assert "hunk recall 100%  (5/5 defective hunks" in out
     assert "fail-on gate ready:" in out
     assert "agent:oracle, preset focused" in out

tests/test_corpus.py

@@ -307,7 +307,66 @@ def test_fail_on_gate_refuses_a_reviewer_that_never_speaks(
 
     assert summary.quiet_rate == 1.0
     assert gate.ready is False
-    assert "recall 0% < 50%" in gate.reason
+    assert "hunk recall 0% < 50%" in gate.reason
+
+
+def test_hunk_recall_counts_any_rule_and_gates_on_it(
+    rubric: ReviewRulesConfig,
+) -> None:
+    from lazycoder.corpus import HunkResult, Label, fail_on_gate, report
+
+    def defective(hid: str, fired: frozenset[RuleId]) -> HunkResult:
+        return HunkResult(
+            hid,
+            Label.DEFECTIVE,
+            frozenset({RuleId.R4}),
+            fired,
+            frozenset(),
+            Verdict.REQUEST_CHANGES,
+            frozenset(),
+        )
+
+    quiet = HunkResult(
+        "C1",
+        Label.CLEAN,
+        frozenset(),
+        frozenset(),
+        frozenset(),
+        Verdict.APPROVE,
+        frozenset(),
+    )
+    # The defect is described under R13, not the labelled R4.
+    results = [
+        quiet,
+        defective("D1", frozenset({RuleId.R13})),
+        defective("D2", frozenset()),
+    ]
+    summary = report(results, rubric)
+
+    assert summary.recall == 0.0  # rule recall: R4 never fired
+    assert summary.hunk_recall == 0.5  # one of two defective hunks flagged
+    assert fail_on_gate(summary, rubric).ready is True
+
+
+def test_out_of_scope_defects_do_not_count_toward_hunk_recall(
+    rubric: ReviewRulesConfig,
+) -> None:
+    from lazycoder.corpus import HunkResult, Label, report
+
+    # A preset that never asks the labelled rule leaves an empty expectation.
+    unscored = HunkResult(
+        "D1",
+        Label.DEFECTIVE,
+        frozenset(),
+        frozenset({RuleId.R7}),
+        frozenset(),
+        Verdict.BLOCK,
+        frozenset(),
+    )
+    summary = report([unscored], rubric)
+
+    assert summary.scored_defective_hunks == 0
+    assert summary.hunk_recall == 1.0
 
 
 def test_corpus_shape_targets_stage2_minimums(tmp_path) -> None: