fix(stats): drop counterfactual-replay rows from the public per-rule precision

src/review/public-rule-precision.ts

@@ -44,6 +44,23 @@ export type PublicRulePrecision = {
   latestBacktestRun: { corpusChecksum: string; at: string } | null;
 };
 
+/**
+ * Provenance tags whose override rows carry a verdict that was NOT a human decision about the rule the row is
+ * filed under, and so cannot support a per-rule "measured accuracy" claim.
+ *
+ * `slop_replay_backfill_v1` (#8277) is the counterfactual replay: it re-scores the deterministic slop signals
+ * over archived diffs, but takes each label verbatim from the `ai_consensus_defect` corpus's human verdict on
+ * the same target (`scripts/backfill-slop-corpus.ts`'s `manifestToSourceCases`). That is exactly the evidence
+ * it was built to be -- internal input to a flip-to-live decision, and by its own module header a LOWER BOUND
+ * on live scoring -- but published under "precision of each automated rule over its human-decided cases" it
+ * asserts something untrue: those were another rule's human-decided cases. It also made the public table print
+ * one number twice, since both rules then shared a label set and a target set.
+ *
+ * NOT excluded: `review_targets_decision_level` (#8083's own backfill), whose labels come from what actually
+ * happened to the PRs THAT rule fired on -- synthesized rows, but genuinely about that rule.
+ */
+const NON_ATTRIBUTABLE_OVERRIDE_PROVENANCES = ["slop_replay_backfill_v1"] as const;
+
 /**
  * Load the public per-rule precision block. Fail-safe per section (the same degradation contract as
  * loadCalibrationTrend): a read error yields an empty/absent section, never a thrown public endpoint.
@@ -57,6 +74,7 @@ export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now
             SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed
        FROM audit_events
       WHERE event_type LIKE '${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}%' AND created_at >= ?
+        AND COALESCE(json_extract(metadata_json, '$.provenance'), '') NOT IN (${NON_ATTRIBUTABLE_OVERRIDE_PROVENANCES.map((tag) => `'${tag}'`).join(", ")})
       GROUP BY rule_id`,
     sinceIso,
   );

test/unit/public-rule-precision.test.ts

@@ -55,6 +55,46 @@ describe("loadPublicRulePrecision (#8230)", () => {
     expect(block.rules).toEqual([{ ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, confirmed: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null }]);
   });
 
+  it("REGRESSION: excludes counterfactual-replay rows whose label came from a DIFFERENT rule's human verdicts", async () => {
+    // #8277's slop replay copies each label verbatim from the ai_consensus_defect corpus's verdict on the same
+    // target, so publishing it under "precision over its human-decided cases" both misattributes the verdict and
+    // made the public table print one number twice (both rules showed decided=460/confirmed=287 in production).
+    const env = createTestEnv();
+    await seedVerdicts(env, "ai_consensus_defect", 15, 5);
+    const store = createSignalStore(env);
+    for (let i = 0; i < 20; i += 1) {
+      await store.recordHumanOverride({
+        ruleId: "slop_gate_score",
+        targetKey: `acme/widgets#${i + 1}`,
+        verdict: i < 15 ? "confirmed" : "reversed",
+        occurredAt: new Date(NOW - 1000 - i).toISOString(),
+        metadata: { backfilled: true, provenance: "slop_replay_backfill_v1" },
+      });
+    }
+
+    const block = await loadPublicRulePrecision(env, NOW);
+    expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75 }]);
+  });
+
+  it("keeps synthesized rows whose labels ARE about the rule they are filed under", async () => {
+    // review_targets_decision_level is #8083's own backfill: the labels come from what happened to the PRs that
+    // rule fired on, so they support a precision claim for it. Only the cross-rule copy is excluded.
+    const env = createTestEnv();
+    const store = createSignalStore(env);
+    for (let i = 0; i < 12; i += 1) {
+      await store.recordHumanOverride({
+        ruleId: "ai_consensus_defect",
+        targetKey: `acme/widgets#${i + 1}`,
+        verdict: i < 9 ? "confirmed" : "reversed",
+        occurredAt: new Date(NOW - 1000 - i).toISOString(),
+        metadata: { backfilled: true, provenance: "review_targets_decision_level" },
+      });
+    }
+
+    const block = await loadPublicRulePrecision(env, NOW);
+    expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 12, confirmed: 9, precision: 0.75 }]);
+  });
+
   it("counts all three reversal shapes over the window and surfaces the latest backtest run's corpus checksum", async () => {
     const env = createTestEnv();
     for (const [eventType, count] of [