diff --git a/src/review/public-rule-precision.ts b/src/review/public-rule-precision.ts index 18b7d9cc7a..e5812ad141 100644 --- a/src/review/public-rule-precision.ts +++ b/src/review/public-rule-precision.ts @@ -44,6 +44,23 @@ export type PublicRulePrecision = { latestBacktestRun: { corpusChecksum: string; at: string } | null; }; +/** + * Provenance tags whose override rows carry a verdict that was NOT a human decision about the rule the row is + * filed under, and so cannot support a per-rule "measured accuracy" claim. + * + * `slop_replay_backfill_v1` (#8277) is the counterfactual replay: it re-scores the deterministic slop signals + * over archived diffs, but takes each label verbatim from the `ai_consensus_defect` corpus's human verdict on + * the same target (`scripts/backfill-slop-corpus.ts`'s `manifestToSourceCases`). That is exactly the evidence + * it was built to be -- internal input to a flip-to-live decision, and by its own module header a LOWER BOUND + * on live scoring -- but published under "precision of each automated rule over its human-decided cases" it + * asserts something untrue: those were another rule's human-decided cases. It also made the public table print + * one number twice, since both rules then shared a label set and a target set. + * + * NOT excluded: `review_targets_decision_level` (#8083's own backfill), whose labels come from what actually + * happened to the PRs THAT rule fired on -- synthesized rows, but genuinely about that rule. + */ +const NON_ATTRIBUTABLE_OVERRIDE_PROVENANCES = ["slop_replay_backfill_v1"] as const; + /** * Load the public per-rule precision block. Fail-safe per section (the same degradation contract as * loadCalibrationTrend): a read error yields an empty/absent section, never a thrown public endpoint. @@ -57,6 +74,7 @@ export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed FROM audit_events WHERE event_type LIKE '${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}%' AND created_at >= ? + AND COALESCE(json_extract(metadata_json, '$.provenance'), '') NOT IN (${NON_ATTRIBUTABLE_OVERRIDE_PROVENANCES.map((tag) => `'${tag}'`).join(", ")}) GROUP BY rule_id`, sinceIso, ); diff --git a/test/unit/public-rule-precision.test.ts b/test/unit/public-rule-precision.test.ts index fdf4e5d85d..a107e788ef 100644 --- a/test/unit/public-rule-precision.test.ts +++ b/test/unit/public-rule-precision.test.ts @@ -55,6 +55,46 @@ describe("loadPublicRulePrecision (#8230)", () => { expect(block.rules).toEqual([{ ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, confirmed: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null }]); }); + it("REGRESSION: excludes counterfactual-replay rows whose label came from a DIFFERENT rule's human verdicts", async () => { + // #8277's slop replay copies each label verbatim from the ai_consensus_defect corpus's verdict on the same + // target, so publishing it under "precision over its human-decided cases" both misattributes the verdict and + // made the public table print one number twice (both rules showed decided=460/confirmed=287 in production). + const env = createTestEnv(); + await seedVerdicts(env, "ai_consensus_defect", 15, 5); + const store = createSignalStore(env); + for (let i = 0; i < 20; i += 1) { + await store.recordHumanOverride({ + ruleId: "slop_gate_score", + targetKey: `acme/widgets#${i + 1}`, + verdict: i < 15 ? "confirmed" : "reversed", + occurredAt: new Date(NOW - 1000 - i).toISOString(), + metadata: { backfilled: true, provenance: "slop_replay_backfill_v1" }, + }); + } + + const block = await loadPublicRulePrecision(env, NOW); + expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75 }]); + }); + + it("keeps synthesized rows whose labels ARE about the rule they are filed under", async () => { + // review_targets_decision_level is #8083's own backfill: the labels come from what happened to the PRs that + // rule fired on, so they support a precision claim for it. Only the cross-rule copy is excluded. + const env = createTestEnv(); + const store = createSignalStore(env); + for (let i = 0; i < 12; i += 1) { + await store.recordHumanOverride({ + ruleId: "ai_consensus_defect", + targetKey: `acme/widgets#${i + 1}`, + verdict: i < 9 ? "confirmed" : "reversed", + occurredAt: new Date(NOW - 1000 - i).toISOString(), + metadata: { backfilled: true, provenance: "review_targets_decision_level" }, + }); + } + + const block = await loadPublicRulePrecision(env, NOW); + expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 12, confirmed: 9, precision: 0.75 }]); + }); + it("counts all three reversal shapes over the window and surfaces the latest backtest run's corpus checksum", async () => { const env = createTestEnv(); for (const [eventType, count] of [