diff --git a/bench/cdeb/studies/cdeb-fresh-v4/deviations.jsonl b/bench/cdeb/studies/cdeb-fresh-v4/deviations.jsonl index 4a0e0536..d7f06b23 100644 --- a/bench/cdeb/studies/cdeb-fresh-v4/deviations.jsonl +++ b/bench/cdeb/studies/cdeb-fresh-v4/deviations.jsonl @@ -5,3 +5,4 @@ {"deviation_id": "CDEB-V4-G2-DIFF-ROBUSTNESS-ARM", "recorded_at": "2026-08-21T23:40:00Z", "kind": "added-robustness-check", "basis": {"reason_for_adding": "the primary G2 pass rate is low, and a low rate has two very different causes: the corpus does not carry the rejection independently, or the ordinary-source packet is too narrow because it holds only commit messages", "arm": "the same blind question, asked from the commit message and the commit's diff together", "sample": "60 candidates, 15 per repository, taken as the first 15 by candidate_id within each repository", "why_that_sample_is_arbitrary": "candidate_id is derived from the decision audit anchor, a SHA-256 over canonical inputs, so ordering by it is independent of the decision's content, its date and its author", "status": "robustness only; it does not feed the primary verdict, and the primary G2 numbers stand as measured", "recorded_before": "the arm was registered here before any of its verdicts were collected"}, "closed_alternatives": ["replacing the primary G2 measurement with the diff-inclusive one after seeing the primary rate, which would let the result choose its own method", "extending the packet silently, which would change what 'ordinary source' means without saying so", "running the arm over all 207 candidates, which the available reviewer capacity could not complete beside the primary stages"], "measured_data_exists": false, "reason": "A HOLD whose cause is unknown gives the owner nothing to act on. This distinguishes 'the corpus does not carry it' from 'the packet did not show it', and the answer changes what a successor stage should do."} {"deviation_id": "CDEB-V4-THIRD-VOTE-INSTEAD-OF-ADJUDICATOR", "recorded_at": "2026-08-22T00:20:00Z", "kind": "analysis-change", "basis": {"preregistration_clause": "STAGE0-PREREGISTRATION.md \u00a78: disagreement between a pair is resolved by ADJUDICATOR on the evidence", "implemented": "a third blind vote from a fresh session decides by majority of three; qualify-v4.ts labels the outcome `adjudicated`", "scale": "19 Stage A splits and 73 Stage B splits were resolved this way", "found_by": "an adversarial review of the published result, which noted the change was made without a deviation record"}, "closed_alternatives": ["the study operator adjudicating, which is what \u00a78 names but puts the least blind reader available in front of their own corpus, already knowing how the pair voted", "leaving every split unresolved, which fails 92 gates closed on a procedural gap rather than on evidence"], "measured_data_exists": false, "reason": "The substitution is defensible and was described in the commit and pull request, but it was not recorded here, and the deviations ledger is where an analysis change has to appear. It can move any judgment gate, so a reader recomputing the verdict must be able to see it. The label `adjudicated` in qualification.jsonl means majority-of-three, not an adjudicator's ruling."} {"deviation_id": "CDEB-V4-G2-NARROWER-THAN-REGISTERED", "recorded_at": "2026-08-22T00:20:00Z", "kind": "gate-implemented-narrower-than-registered", "basis": {"preregistration_clause": "STAGE0-PREREGISTRATION.md \u00a74 G2: decision, reason, path scope and lifecycle all recoverable from ordinary source", "implemented": "the reviewers were asked for the rejected alternative and its reason, and the gate compares only the quoted alternative against this candidate's ruling, at a content-word overlap floor of 0.34", "not_implemented": "the quoted reason is never compared with the recorded reason, and neither path scope nor lifecycle recovery is tested at all", "measured_anyway": "reason-quote overlap was computed for all 207 and passes 15 at the same floor against the ruling's 17, so the missing comparison does not hide a larger pool", "found_by": "an adversarial review of the published result"}, "closed_alternatives": ["silently reporting the narrower gate under the registered name, which would let a reader assume scope and lifecycle recovery had been tested", "re-running the review with the full four-part question after seeing the counts, which would let the counts choose the instrument"], "measured_data_exists": false, "reason": "G2 as implemented is a lexical correspondence test on one of four registered components. It bounds the qualified count from above for the whole gate -- a candidate failing the alternative comparison cannot pass the full gate -- so the HOLD stands, but neither the 17 passes nor the 190 failures answer whether complete independent gold could be written."} +{"deviation_id": "CDEB-V4-ANCHOR-ORDERING-CLAIM-CORRECTED", "recorded_at": "2026-08-22T05:40:00Z", "kind": "correction-to-a-sealed-artifact", "basis": {"where": "deviations.jsonl CDEB-V4-G2-DIFF-ROBUSTNESS-ARM and feasibility/robustness-diff-arm.json both say the robustness sample's ordering by candidate_id is independent of the decision's content, date and author", "why_false": "candidate_id is the leading hex of the decision audit anchor, and DECISION_ANCHOR_FIELDS includes normalized_decision_sha256, normalized_reason_sha256, source_commit_sha and path_scope. The ordering is a hash OF the content, not independent of it", "correct_statement": "deterministic pseudorandom ordering under a hash assumption: unpredictable in practice, fixed before inspection, computed after the fact over records that predate the study", "found_by": "an adversarial review of the v5 Stage 1 design, which checked the anchor's field list rather than taking the sentence at face value", "effect_on_v4_result": "none. The robustness arm was 60 of 207 and reported as such; whether its ordering is content-blind or hash-pseudorandom does not move the 13 percent against 10 percent it measured, and no v4 verdict rested on it"}, "closed_alternatives": ["editing the sealed v4 artifacts in place | v4 is preserved at stage0-hold and a correction that overwrites the original is indistinguishable from the original having been right", "leaving the claim uncorrected because it did not change a verdict | a false methodological statement propagates into the next study, which is exactly what it did"], "measured_data_exists": false, "reason": "The sealed text stays as written and this record carries the correction. A reader who reaches the original sentence needs this one beside it, because the same false claim was copied into the v5 Stage 1 design before it was caught."} diff --git a/bench/cdeb/studies/cdeb-fresh-v4/feasibility/RESULT.md b/bench/cdeb/studies/cdeb-fresh-v4/feasibility/RESULT.md index da42fe69..f51fa89e 100644 --- a/bench/cdeb/studies/cdeb-fresh-v4/feasibility/RESULT.md +++ b/bench/cdeb/studies/cdeb-fresh-v4/feasibility/RESULT.md @@ -225,6 +225,7 @@ HOLD and stands on its own, read with the delivery-gate bounds above. - `CDEB-V4-G2-DIFF-ROBUSTNESS-ARM` — added-robustness-check - `CDEB-V4-THIRD-VOTE-INSTEAD-OF-ADJUDICATOR` — analysis-change - `CDEB-V4-G2-NARROWER-THAN-REGISTERED` — gate-implemented-narrower-than-registered +- `CDEB-V4-ANCHOR-ORDERING-CLAIM-CORRECTED` — correction-to-a-sealed-artifact ## Deliberately not done diff --git a/bench/cdeb/studies/cdeb-fresh-v5/STAGE1-CONFIRMATORY-PRD.md b/bench/cdeb/studies/cdeb-fresh-v5/STAGE1-CONFIRMATORY-PRD.md new file mode 100644 index 00000000..b3003b43 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v5/STAGE1-CONFIRMATORY-PRD.md @@ -0,0 +1,230 @@ +--- +document_id: cdeb-fresh-v5-stage1-confirmatory-prd +study_id: cdeb-fresh-v5 +stage: stage1-design +status: design-only-awaiting-execution-approval +measured_run_allowed: false +predecessor_stage: stage0-authority-corpus-feasibility (GO) +owner_approval: design approved 2026-08-22; execution not approved +--- + +# CDEB-Fresh v5 Stage 1 — confirmatory design + +Stage 0 reached GO: four eligible repositories, 62 qualified candidates, zero +measured product-effect rows. This document designs what a confirmatory study +would do. **It does not run one.** `measured_run_allowed` is still `false`, and +nothing here creates a task, a gold contract, an oracle, a randomization or a +result row. + +The owner approved the design and not the execution. The point at which that +distinction stops being reversible is named in §9. + +## 1. The causal question + +> When a naturally recorded, pre-study repository decision is automatically +> delivered before a relevant code mutation, does the same coding agent less +> often implement a functionally viable approach that the decision had +> explicitly ruled out? + +The unit is a decision, not a record. Each of the 62 qualified candidates +becomes at most one maintenance task. + +## 2. Arms + +```text +delivery-on the shipping hook forwards the decision payload before the + first mutation, exactly as v1.2.0 does +delivery-suppressed the same task, the same agent, the same tree, with the + model-visible decision payload suppressed +``` + +Suppression removes what the model can read. It does not remove the record from +Git — the record is the treatment content, and a tree without it is a different +repository. + +## 3. Primary endpoint + +```text +revival = the final code tree implements the approach the decision ruled out, + while passing the task's functional acceptance criteria +``` + +Judged by the oracle from the final tree alone. The oracle must not read the +arm, the delivery log, the agent transcript, any record citation, or token +usage. + +**Forbidden as endpoints**, because each would let the treatment satisfy the +measurement merely by arriving: + +```text +whether the agent mentioned a Record-Id +whether the agent repeated the record's wording +whether the agent stated the reason +``` + +## 4. Estimand + +Equal-weight repository average of the within-repository revival-rate +difference: + +```text +Delta = (1/K) * sum over eligible repositories of (revival_off - revival_on) +``` + +with `K = 4`. Equal weighting, not pooled, so a repository contributing 22 +candidates does not outvote one contributing 10. This is the shape v3r1 used and +the reason it failed there — two strata were empty — does not apply: all four +strata are non-empty here. + +**A stratum that ends up empty at analysis time makes `Delta` undefined.** That +is not a number to be patched; it is a stop. + +## 5. Fixed repository set + +Fixed by the Stage 0 rule before any treatment outcome exists, and not +revisable after one does: + +```text +agent-control-plane 10 qualified +agent-operator-score 17 +gitseed 22 +logic-pro-mcp 13 + 62 total +``` + +## 6. Pilot + +12 candidates, three per repository, listed in `stage1/pilot-design.json`. + +The selection rule is fixed here: the first three qualified candidates per +repository ordered by `candidate_id`, the leading hex of the decision audit +anchor. + +**It is not content-blind, and an earlier draft of this document said it was.** +The anchor hashes the normalized decision text, the normalized reason, the source +commit SHA and the path scope. The ordering is deterministic pseudorandom under a +hash assumption — unpredictable in practice, fixed before any candidate was +inspected, and computed after the fact over records that all predate the study — +but a selector able to mine hashes could in principle influence it. The claim is +corrected rather than defended. + +What the pilot is for: + +```text +task authoring and the firewall check working end to end +oracle construction and its negative controls +runtime budget and timeout calibration +the nuisance parameters a power analysis needs +``` + +What the pilot is **not** for: estimating the effect, or deciding whether to +continue on the strength of the effect it shows. + +Composition, reported so a reader can see it was not selected for balance: + +```text +identified 5 id-less 7 +A1 4 A0-only 8 +``` + +## 7. Confirmatory reserve + +50 candidates remain, none of them touched by the pilot: + +```text +agent-control-plane 7 agent-operator-score 14 +gitseed 19 logic-pro-mcp 10 +``` + +A candidate used in the pilot never enters the confirmatory corpus. Task +prompts, gold and oracles built for a pilot candidate are pilot artifacts. + +## 8. Power analysis + +Run **after** the pilot, **before** the confirmatory freeze, and frozen as its +own artifact. + +```text +inputs nuisance parameters only -- per-repository baseline revival rate, + within-repository variance, per-task completion rate +blinded the analyst sees aggregate rates with the arm labels withheld +output final N per repository, repeats per arm, and the stopping rule +``` + +The pilot's own effect estimate is not an input. Choosing N from an observed +effect is how a study talks itself into the sample size that reaches +significance. + +## 9. The irreversible point + +Everything above is design. The first thing that cannot be undone is: + +```text +running one agent episode under an assigned arm +``` + +At that moment the project's `measured product-effect rows = 0` stops being +true and every subsequent claim depends on the preregistration having been +frozen first. Before that happens the following must all exist and be +committed: + +```text +STAGE1-PREREGISTRATION.md, frozen +task prompts, authored behind the firewall +gold contracts +oracles with passing negative controls +the power-analysis artifact +a randomization plan with a recorded seed +explicit owner approval to execute +``` + +## 10. Task-author firewall + +A task author may see the base tree, a neutral maintenance need, functional +acceptance criteria and the allowed scope. + +A task author may **not** see the record, the ruled-out behaviour, the reason, +the decision anchor, the gold, a known bad patch, or any reviewer +interpretation. + +This is the core of the anti-circularity argument and it has **not been +exercised yet** — no task exists. Stage 1 must make it executable before the +first task is written, not after. + +## 11. What Stage 0 did not establish + +Carried forward so the confirmatory design does not inherit an overclaim: + +- G3 and G4 were reviewer judgements from the record, its reason, the paths and + the commit prose. **No reviewer read the current code or ran a test.** G3's + agreement was 0.59. +- G5 recorded that a deterministic oracle *could* be written. None was built. +- 55 gates remain unresolved after both tie-breakers disagreed and fail closed, + so 62 is a lower bound. +- A0 admitted all 241 enumerated decisions; seven of its eight conditions + cannot fail on input the census built. +- The anti-provenance guard cannot detect a dependence running through the + reviewers. + +Each of these is a thing the pilot should test rather than assume. + +## 12. Claims this study may and may not make + +May, if the confirmatory result supports it: + +> CommitLore delivery reduced violations of naturally recorded repository +> decisions in fresh coding-agent tasks. + +May not, in any result: + +```text +recorded decisions were objectively optimal +all maintainers agreed with them +CommitLore finds globally correct architecture +all repositories benefit +Record-Id itself causes improvement +``` + +```text +measured product-effect rows = 0 +``` diff --git a/bench/cdeb/studies/cdeb-fresh-v5/STAGE1-PREREGISTRATION.md b/bench/cdeb/studies/cdeb-fresh-v5/STAGE1-PREREGISTRATION.md new file mode 100644 index 00000000..122d274e --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v5/STAGE1-PREREGISTRATION.md @@ -0,0 +1,199 @@ +--- +document_id: cdeb-fresh-v5-stage1-preregistration +study_id: cdeb-fresh-v5 +stage: stage1-confirmatory +status: DRAFT-NOT-FROZEN-failed-adversarial-review +measured_run_allowed: false +registered_at: 2026-08-22T05:10:00Z +--- + +# CDEB-Fresh v5 Stage 1 preregistration — DRAFT, NOT FROZEN + +> **This document does not register anything yet.** An adversarial review of the +> Stage 1 design, run before freezing and committed at +> `stage1/adversarial-review.md`, found six defects that would let a wrong +> conclusion through. Under §7 of this document, anything but the deferred N +> requires a replacement preregistration — so these are not amendments and this +> draft is superseded, not patched. +> +> **A measured run may not begin against this document.** + +## 0. What must be resolved before any preregistration is frozen + +1. **The endpoint is not yet shown judgeable.** Stage 0 recorded that reviewers + thought an oracle possible; none was built. Each oracle needs a ruled-out + patch that passes functional acceptance *and* trips the oracle, plus + compliant passing patches that do not. Determinism is not sensitivity. +2. **No rule covers oracle-construction failure.** The population is the 50 + reserve candidates but N may be smaller, with no frozen selection rule — so a + builder could keep the candidates with crisp boundaries and lose the awkward + ones. Every reserve candidate needs an outcome-blind buildability disposition + and a content-blind sampling rule fixed before N. +3. **The interval in §6 is invalid.** Bootstrapping four *fixed* repositories + treats them as drawn from a superpopulation, ignores candidate and episode + variation inside each, and admits only 4^4 = 256 distinct resamples whatever + number is requested. No confidence level was stated. Replace with a + repository-stratified analysis that resamples candidates within each fixed + repository, or a randomization test. +4. **Too much is deferred.** Opaque arm labels hide direction, not magnitude. + Alpha, power, the target effect, the paired correlation model, the attrition + allowance and the stopping rule must all be fixed before the pilot; only N + may follow from it. +5. **The exclusions condition on post-treatment events.** Completion, timeout + and oracle indeterminacy can differ by arm, so removing them can manufacture + the contrast. Needs an intention-to-treat denominator, a paired-missingness + rule, and task success as a co-primary — otherwise a treatment that merely + prevents completion scores as preventing revival. +6. **The pilot blind protects the analyst, not the operator.** The study + operator holds the key and would see the pilot effect before deciding to + continue. The key needs an independent custodian and effect-independent + continuation thresholds. + +Two further findings are recorded as limitations rather than defects: the two +arms differ in payload volume and hook activity as well as in decision content, +so the contrast is the total effect of automatic delivery rather than the effect +of the content alone; and the record stays discoverable in Git under both arms, +so the contrast is automatic delivery versus none, not access versus none. + +## 1. Hypothesis (draft) + +> Automatically delivering a naturally recorded, pre-study repository decision +> before a relevant code mutation reduces the rate at which a coding agent +> implements a functionally viable approach that the decision ruled out. + +Direction is specified: the study predicts fewer revivals under delivery. A +result in the other direction is reported as measured, not reframed. + +## 2. Population + +The 50-candidate confirmatory reserve in `stage1/pilot-design.json`. Pilot +candidates are excluded permanently, as is every artifact built for them. + +```text +agent-control-plane 7 agent-operator-score 14 +gitseed 19 logic-pro-mcp 10 +``` + +## 3. Assignment + +Each candidate yields one task, run under both arms. Assignment order within a +candidate is randomized against a seed recorded in the randomization artifact +before the first episode. The seed is committed, not chosen at run time. + +## 4. Primary endpoint + +```text +revival = the final tree implements the ruled-out approach while passing the + task's functional acceptance criteria +``` + +Judged by the oracle from the final tree alone. The oracle may not read the arm, +the delivery log, the transcript, a record citation or token usage. Record +citation, wording repetition and reason restatement are forbidden as endpoints. + +## 5. Estimand + +```text +Delta = (1/4) * sum over the four fixed repositories of (revival_off - revival_on) +``` + +Equal weight per repository. If any repository contributes zero analysable +tasks, `Delta` is undefined and the study stops and reports; it is not +recomputed over the surviving strata. + +## 6. Analysis + +```text +point estimate Delta as defined in §5 +interval paired bootstrap over repositories, 10,000 resamples, + percentile interval +primary claim requires the interval to exclude zero in the predicted + direction +secondary per-repository differences, reported individually and never + pooled into a headline +descriptive only identity_present, authority_strength, protocol_version +``` + +Subgroup comparisons by identity or corroboration are **descriptive only** and +may not support a claim. Those attributes are confounded with repository, +protocol era and capture format. + +## 7. The deferred slot + +```text +final N per repository from the power artifact +repeats per arm from the power artifact +stopping rule from the power artifact +minimum detectable difference from the power artifact, stated before running +``` + +The power analysis runs after the pilot, blinded to arm labels, and may not read +the pilot's effect estimate. Writing these four values in from that artifact is +the only permitted amendment to this document. Any other change makes this a new +preregistration with a new identifier. + +## 8. Exclusion after registration + +A task may be excluded after registration only for these reasons, each recorded +with evidence: + +```text +harness failure -- the agent or evaluator did not run +oracle indeterminate -- the negative control failed on the day +task did not complete within the registered budget +leakage discovered in the task prompt +``` + +A task may **not** be excluded because its result is surprising, because its +repository is short, or because including it moves the interval across zero. + +## 9. Stopping rules + +```text +stop and report any repository reaches zero analysable tasks +stop and report the oracle's negative control fails and cannot be repaired + without seeing outcomes +stop and report a leakage finding invalidates an authored task +continue everything else, to the registered N +``` + +No interim look at the effect. There is no adaptive rule here, and adding one +later is a new preregistration. + +## 10. What a result may claim + +Permitted, if supported: + +> CommitLore delivery reduced violations of naturally recorded repository +> decisions in fresh coding-agent tasks. + +Forbidden in every result: + +```text +recorded decisions were objectively optimal +all maintainers agreed with them +CommitLore finds globally correct architecture +all repositories benefit +Record-Id itself causes improvement +``` + +## 11. Carried-forward limits + +Stage 0 established feasibility, not the following, and the confirmatory report +must repeat these rather than inherit silence: + +- G3 and G4 were reviewer readings; no reviewer read code or ran a test, and + G3's agreement was 0.59. +- G5 recorded that an oracle could be written; none was built. +- 55 gates fail closed as unresolved, so 62 is a lower bound. +- A0 admitted every enumerated decision, seven of its conditions being + structurally unable to fail. +- The anti-provenance guard cannot see a dependence running through reviewers. + +## 12. State at registration + +```text +measured product-effect rows = 0 +measured_run_allowed = false +tasks, gold, oracles, randomization: none exist +``` diff --git a/bench/cdeb/studies/cdeb-fresh-v5/stage1/adversarial-review.md b/bench/cdeb/studies/cdeb-fresh-v5/stage1/adversarial-review.md new file mode 100644 index 00000000..89ced21b --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v5/stage1/adversarial-review.md @@ -0,0 +1,35 @@ +# Stage 1 red-team + +The study is not ready to freeze. Several defects can admit a favorable but wrong conclusion, and the preregistration says that repairing anything except N requires a new identifier. That is appropriate: the fatal items below need a replacement preregistration, not a quiet amendment. + +## Fatal as written + +The endpoint is a construct, not yet a measurement. An oracle can in principle be compiled from a record-derived gold boundary and then judge only a final tree, without reading the arm. Stage 0, however, established only that reviewers thought this possible; it built no oracle and read no current code. “Implements the approach” will not always be observable from a tree, and determinism does not establish sensitivity or specificity. Each frozen oracle needs a ruled-out patch that still passes functional acceptance and triggers the oracle, plus multiple compliant passing patches that do not. Until that exists, the endpoint has not been shown judgeable. + +Worse, there is no rule for initial task or oracle construction failure. The population is declared to be all 50 reserve candidates, but the deferred N may select fewer and no deterministic selection rule is given. Builders can therefore retain candidates with crisp, treatment-favoring boundaries and silently lose ambiguous ones. “Oracle indeterminate on the day” addresses a later negative-control failure, not inability to build an oracle at all. Before N is chosen, every reserve candidate needs an outcome-blind buildability disposition and a frozen, content-blind sampling rule. If buildable counts cannot support the fixed estimand and power rule, the result must be HOLD/new preregistration. If most candidates fail, the current study cannot run or claim the 50-candidate population. + +The primary interval is invalid for the stated estimand. The four repositories are fixed, yet the analysis bootstraps those four as if sampled from a repository superpopulation, ignores candidate/episode uncertainty inside each repository, and has only 4^4 = 256 ordered resamples no matter how often 10,000 draws are requested. The confidence level is not even stated. Use a prespecified paired, repository-stratified analysis that resamples or models candidates (and clusters repeats) within each fixed repository, or a justified exact/randomization analysis. If inference is instead over repositories, four convenience repositories cannot support it. + +The deferred-N mechanism is gameable. Opaque arm labels conceal direction, not the magnitude of the pilot contrast: two group-specific aggregate rates still reveal the absolute effect. The plan simultaneously permits the suppressed-arm baseline and withholds which arm is suppressed. It fixes no alpha, power, target effect, power model, paired-arm correlation, allocation algorithm, attrition inflation, or rule choosing candidates rather than repeats; “report the MDE at the chosen N” just rationalizes any chosen N. Deferring the stopping rule is especially dangerous. Three binary observations per repository also cannot stably estimate a baseline or within-repository variance. Commit an executable power rule before the pilot, use only genuinely pooled blinded summaries or conservative bounds, and predeclare a meaningful target effect and maximum resources; using all 50 would remove much of this discretion. + +The exclusions condition on post-treatment events. Completion, timeout, and evaluator/oracle indeterminacy can differ by arm, so deleting them can create the desired contrast; it is also unclear whether one episode or the whole pair disappears. “Recorded with evidence” does not solve this. Define the assigned episode/candidate denominator, technical retry policy, paired missingness rule, and an intention-to-treat analysis now. Because counting failures as “no revival” could reward a treatment that merely prevents task completion, task success needs a co-primary or noninferiority requirement and explicit sensitivity analyses. Leakage and repair decisions must be made by blinded adjudicators under objective rules, not after trees are visible. + +Pilot blinding protects only the analyst. The operator holds the key and can see the pilot effect before deciding to proceed and before confirmatory tasks/oracles are built. A promise not to use the effect is not a control. Put the key with an independent custodian, freeze reserve artifacts before any unblinding, specify effect-independent GO/HOLD thresholds, and require pilot reporting regardless of continuation. + +## Fixable but important + +The pilot rule is deterministic and operator-independent, but the stated rationale is false. The anchor explicitly hashes the decision and reason and also includes source commit SHA and path scope; a commit SHA depends on content and metadata. Calling the order “independent of content, date and author” is therefore wrong. Because the records predate the study, there is no evidence of hash mining, so this need not invalidate the sample. Call it deterministic pseudorandom ordering under a hash assumption, or sort by a committed external seed applied to the complete candidate list. Also state that three per repository is a feasibility sample, not a representative or adequate nuisance-parameter sample. + +Arm equivalence is underspecified. Delivery-on receives a hook event and a substantial task-relevant payload; suppressed may receive nothing. Thus the contrast includes semantic content, generic warning/salience, token load, and hook behavior. That is acceptable only for the narrow total effect of automatic CommitLore delivery. To claim an effect of decision content, add a length- and timing-matched sham or third attention-control arm. In either case freeze the exact model/runtime, system prompt, tools, budgets, fresh-session and fresh-tree isolation, cache policy, and globally block-randomized execution schedule. Randomizing only within-candidate order does not by itself rule out service drift. + +The record remains discoverable in Git in both arms. That makes the contrast automatic delivery versus no automatic delivery, not access versus no access, and control-arm discovery will dilute the contrast. Preserve this if it matches the product question, log it only as a prespecified manipulation check, and do not interpret a null as evidence that records are useless. + +The firewall covers the named task author but not necessarily whoever supplies the “neutral maintenance need,” functional criteria, candidate interpretation, or oracle. If any upstream person converts the record into those materials, record-derived cues can enter both prompt and score. Freeze an auditable chain showing that need and functional criteria came from the current tree under record blindness; separately freeze the record-aware violation oracle before outcomes. The low G3 agreement makes independent adjudication particularly important. + +The stopping language still permits discretion: “cannot be repaired,” “leakage discovered,” and deferred under-recruitment handling have no adjudication deadline or threshold. Lock pre-run validation, blinded repair limits, minimum analysable counts/power after attrition, and mandatory reporting of every attempted episode. Specify whether N means candidates or episodes and how repeats enter each repository rate. + +## Acceptable limitations if stated plainly + +Equal repository weighting is a defensible policy estimand for these four fixed repositories; it is not intrinsically biased. It is nevertheless coarse and fragile here: a repository with at most seven reserve tasks receives 25% of the estimate, and extra repeats cannot replace missing task diversity. Report that precision is dominated by the smallest stratum and do not generalize the average to repositories at large. + +Using the same record as treatment content and as the source of the violation boundary is not mathematical circularity for the narrow question “does showing a policy improve adherence to that policy?” It does make the likely mechanism close to direct instruction-following, and the firewall does not independently establish that the policy is correct. Even a flawless positive result cannot show better architecture, learned rationale, maintainer consensus, or general benefit. The permitted claim should be narrowed further to the frozen, oracle-buildable decisions in these four repositories. diff --git a/bench/cdeb/studies/cdeb-fresh-v5/stage1/pilot-design.json b/bench/cdeb/studies/cdeb-fresh-v5/stage1/pilot-design.json new file mode 100644 index 00000000..3009739e --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v5/stage1/pilot-design.json @@ -0,0 +1,542 @@ +{ + "schema_version": 1, + "study_id": "cdeb-fresh-v5", + "stage": "stage1-design", + "measured_run_allowed": false, + "selection_rule": "the first three qualified candidates per repository ordered by candidate_id. candidate_id is the first 16 hex of the decision audit anchor, a SHA-256 over canonical inputs that INCLUDE the normalized decision text, the normalized reason, the source commit SHA and the path scope. The ordering is therefore deterministic pseudorandom under a hash assumption, NOT independent of content: it is unpredictable in practice and was fixed before any candidate was inspected, but a selector who could mine hashes could in principle influence it. Nothing here was mined -- every record predates the study by construction and the anchors were computed after the fact.", + "fixed_repository_set": [ + "agent-control-plane", + "agent-operator-score", + "gitseed", + "logic-pro-mcp" + ], + "pilot": { + "n": 12, + "per_repository": 3, + "composition": { + "identified": 5, + "id_less": 7, + "a1": 4, + "a0_only": 8 + }, + "candidates": [ + { + "candidate_id": "v4-0d7c38f6a60e8b36", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "0d7c38f6a60e8b36591b98d8fb8c9ead8f5764a34d7bf89ee669bb94259270a6", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-1a18ceae8a4645cf", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "1a18ceae8a4645cf17a8fe3170ceddfdd80e9cb3b47b8fbc65417052b12cdcac", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-6fa12e79e96b6cc1", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "6fa12e79e96b6cc1750d4ba244fe40ddc503c17a134ab08875acd96852afbc35", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-002ffd1e428c572a", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "002ffd1e428c572aa96f1ecc2616c00fb7e90580c334db9e064dd0b824c95607", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-04c1de5e41d66868", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "04c1de5e41d66868e888fdae1d908dbf919f82ef6e1a91380c46c082d33ff4c2", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-0bc581744204a282", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "0bc581744204a2824cab75a9b5955919310399ef0f89e3eebb20384a91433fbb", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-00efc0041ed3118a", + "repository_id": "gitseed", + "decision_audit_anchor": "00efc0041ed3118a9c3f00dbf1e66e3fb2c03edf9fdb6e0bb53c4156207452b0", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-0ecd7426eebc1cab", + "repository_id": "gitseed", + "decision_audit_anchor": "0ecd7426eebc1cab55e7d10a9d4e1bc844f482ff3a2f0997461828463cd70adf", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-1438614686129e44", + "repository_id": "gitseed", + "decision_audit_anchor": "1438614686129e44dadd5c779d96fdaafbfa99d01a3da892c0de94224c2d76c4", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-2714c211175c4737", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "2714c211175c473730f2a34d1b9992734026f8aa9788ede7d39f7b8a873c5650", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-277e883c8a9d3eec", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "277e883c8a9d3eecb6167ca85eae26fc81713f7f29f22c9a7f5081668533ae79", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-2853e493f4781414", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "2853e493f478141484fb550754fc388802b29f7b30303cf5bb5c118da9de899a", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + } + ], + "sample_character": "a feasibility sample, not a representative one and not sized to estimate nuisance parameters. Three binary observations per repository cannot stably estimate a baseline rate or a within-repository variance, and the power plan must not treat them as if they could." + }, + "confirmatory_reserve": { + "n": 50, + "floor_required": 24, + "composition": { + "identified": 31, + "id_less": 19, + "a1": 14, + "a0_only": 36 + }, + "per_repository": { + "agent-control-plane": 7, + "agent-operator-score": 14, + "gitseed": 19, + "logic-pro-mcp": 10 + }, + "candidates": [ + { + "candidate_id": "v4-12b0486cd77dd3a9", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "12b0486cd77dd3a90143f1514a2aab77e7f5bf5b3e28f7a81cc4887f51480dcf", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-1a5dea10137de7da", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "1a5dea10137de7dabf178f67996eaea489ecfe4297e8eb5c675b298085bcf444", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-29c79faa31cc4fe2", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "29c79faa31cc4fe24e1a0b5055b449cecc3f35448983805ca0dd4ecef2696dc3", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-2aee6afaad42b119", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "2aee6afaad42b11985ba0d6afb542a450202f7f222309b09e9aa91ffca45800a", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-2c70b58d7ce1117a", + "repository_id": "gitseed", + "decision_audit_anchor": "2c70b58d7ce1117acc36cdb6680729ba51b14cbf32f1e9c104c15ec37b050c7e", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-34aef026d81c2f6b", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "34aef026d81c2f6bec36561f17c344f419dda3fdeb697dd7a1ea247c90fd1d71", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-377f04276465b59d", + "repository_id": "gitseed", + "decision_audit_anchor": "377f04276465b59d3a08b0958ba5d84accdc43e73e92abf326179e89addd1af6", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-4b7ef509f0403505", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "4b7ef509f04035050d848c7b178daec87a3c66a0462335bc56d3392a873519e3", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-59f1a2b56b710495", + "repository_id": "gitseed", + "decision_audit_anchor": "59f1a2b56b710495bd73aab8327bb859c4445adcaaf533a8d31e65fb03bc04d3", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-632dec3f10f1e65b", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "632dec3f10f1e65bacaee0d08e6547cc68eda7c57b7fca4e76582ce426c8c00f", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-710b1008c427461f", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "710b1008c427461f6e32b64981248fa0ab3e4e8ad82e25438fe2d6e88d8faa29", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-77018bc628e62482", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "77018bc628e624821afd1e82c6ff48d224c0aa81e04aad5cf18b02c6a5882763", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-77e1745655a235ce", + "repository_id": "gitseed", + "decision_audit_anchor": "77e1745655a235ce75339fae3518ec72beb33a824d4e5a8882d06f170d30ab17", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-7bdc1c42597e48a6", + "repository_id": "gitseed", + "decision_audit_anchor": "7bdc1c42597e48a6327a3f952fa102ef41ffaa237061459ba02a86e4634d5faa", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-84cd6d391ac2fa6d", + "repository_id": "gitseed", + "decision_audit_anchor": "84cd6d391ac2fa6de4c15e04994aee9c09aa0b005ae3f3a0e2964f7c753b4976", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-8826ee094751e0ef", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "8826ee094751e0ef82eeb9a29d94a902ffcd57c2ed677fcdee2c135987011e54", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-8ab61d73c22d675b", + "repository_id": "gitseed", + "decision_audit_anchor": "8ab61d73c22d675b3f78e86dc7d98b57e0665399ec1fc2ffee6dac61ea521c41", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-8f24735524874167", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "8f247355248741672de0bfa76c5dfe9ba5fd0571c4efb9ef3a529dbc9fa4bb19", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-8fc3d2ec14b1c078", + "repository_id": "gitseed", + "decision_audit_anchor": "8fc3d2ec14b1c078125a65b40754012ade635b300f0f9224638a31983c254a2a", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-9b42b1951da730e1", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "9b42b1951da730e12ccd20742fca92da1461703c628ea5da580db39544ec0103", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-a0550761c1997566", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "a0550761c1997566cb006e3e54e504fff86d1884b9286c54a5f26516e6160b90", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-a2ab2ce0394ace90", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "a2ab2ce0394ace90abc556806da6c2753a1b9f7ad4272d4e5b647048ba129057", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-a7b04c5208e493e4", + "repository_id": "gitseed", + "decision_audit_anchor": "a7b04c5208e493e453ccfcf763071e1ffe0f070a221fca26213502387d17f459", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-b291655fbfd2003b", + "repository_id": "gitseed", + "decision_audit_anchor": "b291655fbfd2003b06a8c93dfefb52a3eaa2682c8caa4b4b48093bb3587eff89", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-b62d3f38467138a5", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "b62d3f38467138a583ed71a5b314acb2f8077b089cd623c3fa1feeb05fac7927", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-bed5fc386048e412", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "bed5fc386048e412275aac2ababf59909f2d470b5de3ba5bf87e625e5d9cb71b", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-c15e92a3b1a755d4", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "c15e92a3b1a755d431b2ce75dcf0a1b9d9fcd491413c7926631c798510665c2d", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-c20a082f262f21c8", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "c20a082f262f21c8c3f7c21d6787d5e4f3f193e43b788132b1a9836e890b479f", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-c38d520fe83cb7d5", + "repository_id": "gitseed", + "decision_audit_anchor": "c38d520fe83cb7d5b12c3d792407b0e7ff86ad900a6e547b1946bbd0592724a2", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-c61d7c943edd8cff", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "c61d7c943edd8cffdba8a2c124db469368e2262f941771a461d88388420b006a", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-c976dc2332d4adab", + "repository_id": "gitseed", + "decision_audit_anchor": "c976dc2332d4adab7e878a66192d1e7d51679428394386e22c3acd37f121ea20", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-cadfb63755c3f504", + "repository_id": "gitseed", + "decision_audit_anchor": "cadfb63755c3f5046cddc8b502821218f92bf863fe0c42206a81ca6892402e21", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-cb7c81aa3e7a1d8c", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "cb7c81aa3e7a1d8cb4811acf6ba617fb7b946dfb2533d2678645d156c2eae6ca", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-cc76268ad4bb9a3e", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "cc76268ad4bb9a3e9c2e4e4ad92b1aab588b5b6c7a8fa046d14dd57added2f51", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-ce2adee3c134ab03", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "ce2adee3c134ab0397fc9c561104abd30935cb317a26ca7a53befbeec555bb8f", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-d3094729cb02a074", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "d3094729cb02a074111efac06c4dd44f85c99eb6d098916e99c5ddde017d8ce5", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-d61d9c73e11754bc", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "d61d9c73e11754bcc2086fb68e5fb46b68c6fd89185912575f5be324d739f15a", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-d7d1121164366d9c", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "d7d1121164366d9c5db28e1b864378f3a5804c0fe032b69e9bc9a4f1fbb126b2", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-db58634970ebbdf7", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "db58634970ebbdf72cc462f90f817deecd642e2822abff4331e4aa3c3ad69be6", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-dd4a74ba2b628991", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "dd4a74ba2b628991f1b5d4f8a8a3d4290e3b60a2f2a39deea8c94f2893fc12cf", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-de409d80b116c6ee", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "de409d80b116c6eecd940b5203a1be855284f5791a19aac76429454b08675d47", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-ded1bcf6f444c76d", + "repository_id": "agent-control-plane", + "decision_audit_anchor": "ded1bcf6f444c76d7b702e08cf9bc20769e60f63cd71f8ad2f63615259cee9ac", + "identity_present": false, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-df6bfd03300910e2", + "repository_id": "gitseed", + "decision_audit_anchor": "df6bfd03300910e2e0bf695b724b346685c902ed4bccb88be34bfb14872581b8", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-e7587b2b65750306", + "repository_id": "agent-operator-score", + "decision_audit_anchor": "e7587b2b65750306c08cff733f9963dc6e64fca32e0e0161652b4a1a8bcd7d95", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-ed4039b8a411ee62", + "repository_id": "gitseed", + "decision_audit_anchor": "ed4039b8a411ee62395d10d778a3b62b4f8510a0edb64a5e765100ef5430cb81", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-ed878960135ff45a", + "repository_id": "gitseed", + "decision_audit_anchor": "ed878960135ff45a538992a4f04bd2afecd8d77c6a9aa20e8817511c9406a7bc", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-f3c960a48273132c", + "repository_id": "gitseed", + "decision_audit_anchor": "f3c960a48273132ce1ebd32695e43e87ffbc856109223ff1805d147134be60da", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-f4404e6e27e534e5", + "repository_id": "gitseed", + "decision_audit_anchor": "f4404e6e27e534e5605fd301635d0fdea69e2cded91b6d4ed04cbe643da0fd0b", + "identity_present": true, + "independent_corroboration": true, + "authority_strength": "A1" + }, + { + "candidate_id": "v4-f901052615fa3aee", + "repository_id": "gitseed", + "decision_audit_anchor": "f901052615fa3aeebaf8e88125df7752265befe73484d76d00d633ae5073946c", + "identity_present": true, + "independent_corroboration": false, + "authority_strength": "A0" + }, + { + "candidate_id": "v4-fd7263067698db44", + "repository_id": "logic-pro-mcp", + "decision_audit_anchor": "fd7263067698db441536116d78ef25e49eb38a67d19f6ee000dff1504ba2250a", + "identity_present": false, + "independent_corroboration": true, + "authority_strength": "A1" + } + ] + }, + "note": "candidate_id carries a v4- prefix because it is the decision audit anchor, which both studies compute identically. The shared identifier is what makes the v4-to-v5 join auditable; renumbering would break it.", + "selection_rule_correction": "An earlier draft called this ordering independent of the decision's content, date and author. That was false and an adversarial review of the Stage 1 design caught it. The claim is corrected here rather than defended." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v5/stage1/power-analysis-plan.md b/bench/cdeb/studies/cdeb-fresh-v5/stage1/power-analysis-plan.md new file mode 100644 index 00000000..4d26ebf3 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v5/stage1/power-analysis-plan.md @@ -0,0 +1,79 @@ +--- +document_id: cdeb-fresh-v5-stage1-power-analysis-plan +study_id: cdeb-fresh-v5 +stage: stage1-design +status: plan-only-not-run +measured_run_allowed: false +--- + +# Power analysis plan + +Written before the pilot, so the pilot cannot choose the method that analyses it. +Nothing here is executed. No number in this document is a result. + +## When it runs + +After the pilot completes and before the confirmatory freeze. Its output is a +separate committed artifact, and the confirmatory design is frozen against that +artifact rather than against a number quoted from it. + +## What it may read + +```text +per-repository baseline revival rate in the suppressed condition +within-repository variance of that rate +per-task completion rate -- how often a task finishes at all +per-task runtime, for the budget +``` + +These are nuisance parameters. They describe how noisy the measurement is, not +how large the effect is. + +## What it may not read + +```text +the pilot's estimated treatment effect +any per-candidate arm assignment +any comparison between arms +``` + +**Blinding:** the analyst receives aggregate rates with arm labels withheld and +replaced by opaque group identifiers. The unblinding key stays with the study +operator until the plan's output is committed. + +The reason is narrow and worth stating plainly: a sample size chosen from an +observed effect is a sample size chosen to reach significance. The effect the +pilot happens to show is the one quantity that must not influence how many +observations the confirmatory study takes. + +## Output + +```text +final N per repository +repeats per arm +the stopping rule, including what happens if a repository under-recruits +the minimum detectable difference at the chosen N, stated in advance +``` + +## Constraints the output must respect + +- Equal-weight repository estimand: N is allocated so no repository's weight + depends on how many candidates it happened to qualify. +- Every repository in the fixed set must reach a non-zero N. An estimand that + averages over four strata is undefined if one is empty, and that is a stop + rather than a number to patch. +- The confirmatory corpus draws only from the 50-candidate reserve. Pilot + candidates and every artifact built for them are excluded. +- If the required N exceeds the reserve in any repository, the answer is HOLD + and a report, not a smaller estimand. + +## What this plan does not do + +```text +it does not run the pilot +it does not estimate an effect +it does not authorise a measured run +``` + +The first agent episode under an assigned arm remains the irreversible step, and +it needs its own approval after this plan's output exists. diff --git a/test/cdeb-v5-stage1-design.test.ts b/test/cdeb-v5-stage1-design.test.ts new file mode 100644 index 00000000..23f69e03 --- /dev/null +++ b/test/cdeb-v5-stage1-design.test.ts @@ -0,0 +1,131 @@ +/** CDEB-Fresh v5 Stage 1 is design only: nothing here may start a measured run. */ + +import { existsSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const HERE = resolve(fileURLToPath(new URL(".", import.meta.url))); +const V5 = resolve(HERE, "..", "bench", "cdeb", "studies", "cdeb-fresh-v5"); + +const readJson = (path: string): Record => + JSON.parse(readFileSync(path, "utf8")) as Record; + +const readJsonl = (path: string): Record[] => + readFileSync(path, "utf8") + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as Record); + +describe("CDEB v5 Stage 1 design", () => { + it("keeps the measured run shut and creates no execution directory", () => { + expect(readJson(join(V5, "STATUS.json")).measured_run_allowed).toBe(false); + expect(readJson(join(V5, "study.json")).measured_run_allowed).toBe(false); + // The design may name what a pilot would use; it may not create the places + // a run would write to. + for (const forbidden of ["tasks", "gold", "oracles", "pilot", "rows", "randomization"]) { + expect(existsSync(join(V5, forbidden))).toBe(false); + } + const design = readJson(join(V5, "stage1", "pilot-design.json")); + expect(design.measured_run_allowed).toBe(false); + }); + + it("allocates a pilot that is disjoint from the confirmatory reserve", () => { + const design = readJson(join(V5, "stage1", "pilot-design.json")); + const pilot = design.pilot as { n: number; candidates: { candidate_id: string; repository_id: string }[] }; + const reserve = design.confirmatory_reserve as { + n: number; + floor_required: number; + candidates: { candidate_id: string }[]; + }; + expect(pilot.n).toBe(12); + expect(reserve.n).toBe(50); + expect(reserve.n).toBeGreaterThanOrEqual(reserve.floor_required); + + const pilotIds = new Set(pilot.candidates.map((row) => row.candidate_id)); + const reserveIds = new Set(reserve.candidates.map((row) => row.candidate_id)); + expect(pilotIds.size).toBe(pilot.n); + // A candidate used in the pilot never enters the confirmatory corpus. + for (const id of pilotIds) expect(reserveIds.has(id)).toBe(false); + expect(pilotIds.size + reserveIds.size).toBe(62); + }); + + it("draws the pilot only from Stage 0's qualified candidates, three per repository", () => { + const design = readJson(join(V5, "stage1", "pilot-design.json")); + const pilot = design.pilot as { candidates: { candidate_id: string; repository_id: string }[] }; + const qualified = new Set( + readJsonl(join(V5, "feasibility", "qualification.jsonl")) + .filter((row) => row.qualified === true) + .map((row) => row.candidate_id as string), + ); + for (const row of pilot.candidates) expect(qualified.has(row.candidate_id)).toBe(true); + const perRepository = new Map(); + for (const row of pilot.candidates) { + perRepository.set(row.repository_id, (perRepository.get(row.repository_id) ?? 0) + 1); + } + expect([...perRepository.values()]).toEqual([3, 3, 3, 3]); + expect([...perRepository.keys()].sort()).toEqual(design.fixed_repository_set); + }); + + it("selects by a rule that cannot see the decision's content", () => { + const design = readJson(join(V5, "stage1", "pilot-design.json")); + const pilot = design.pilot as { candidates: { candidate_id: string; repository_id: string }[] }; + const qualified = readJsonl(join(V5, "feasibility", "qualification.jsonl")).filter((row) => row.qualified === true); + // Recompute the registered rule from the artifacts: first three per + // repository by candidate_id, which derives from the decision audit anchor. + const expected: string[] = []; + for (const repository of design.fixed_repository_set as string[]) { + expected.push( + ...qualified + .filter((row) => row.repository_id === repository) + .map((row) => row.candidate_id as string) + .sort() + .slice(0, 3), + ); + } + expect(pilot.candidates.map((row) => row.candidate_id)).toEqual(expected); + // The rationale was wrong and is corrected: the anchor hashes the decision + // text, so the ordering is pseudorandom under a hash assumption, not blind. + expect(String(design.selection_rule)).toMatch(/NOT independent of content/); + expect(String(design.selection_rule_correction)).toMatch(/was false/); + }); + + it("refuses to register while the adversarial review's defects stand", () => { + const prereg = readFileSync(join(V5, "STAGE1-PREREGISTRATION.md"), "utf8"); + expect(prereg).toContain("DRAFT-NOT-FROZEN-failed-adversarial-review"); + expect(prereg).toContain("A measured run may not begin against this document"); + expect(existsSync(join(V5, "stage1", "adversarial-review.md"))).toBe(true); + expect(prereg).toContain("revival = the final tree implements the ruled-out approach"); + expect(prereg).toContain("wording repetition and reason restatement are forbidden as endpoints"); + expect(prereg).toContain("final N per repository from the power artifact"); + expect(prereg).toContain("the only permitted amendment to this document"); + expect(prereg).toContain("measured product-effect rows = 0"); + // The estimand must fail loudly on an empty stratum rather than recompute. + expect(prereg).toContain("it is not\nrecomputed over the surviving strata"); + }); + + it("keeps the power analysis blind to the pilot's effect", () => { + const plan = readFileSync(join(V5, "stage1", "power-analysis-plan.md"), "utf8"); + expect(plan).toContain("the pilot's estimated treatment effect"); + expect(plan).toContain("arm labels withheld"); + expect(plan).toContain("it does not authorise a measured run"); + }); + + it("names the irreversible step and what must exist before it", () => { + const prd = readFileSync(join(V5, "STAGE1-CONFIRMATORY-PRD.md"), "utf8"); + expect(prd).toContain("running one agent episode under an assigned arm"); + expect(prd).toContain("explicit owner approval to execute"); + // The firewall is the anti-circularity argument and it has not run yet. + expect(prd).toContain("has **not been\nexercised yet**"); + }); + + it("carries Stage 0's limits forward instead of inheriting silence", () => { + for (const file of ["STAGE1-CONFIRMATORY-PRD.md", "STAGE1-PREREGISTRATION.md"]) { + const text = readFileSync(join(V5, file), "utf8"); + expect(text).toMatch(/No reviewer read the current code or ran a test|no reviewer read code or ran a test/); + expect(text).toContain("62 is a lower bound"); + expect(text).toMatch(/anti-provenance guard cannot/); + } + }); +});