diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 4ef08d1c6a..ce71f46e5b 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -576,6 +576,78 @@ "windowDays", "gamingFlagsCaught" ] + }, + "rulePrecision": { + "type": "object", + "properties": { + "windowDays": { + "type": "number" + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "ruleId": { + "type": "string" + }, + "decided": { + "type": "number" + }, + "precision": { + "type": "number", + "nullable": true + } + }, + "required": [ + "ruleId", + "decided", + "precision" + ] + } + }, + "reversals": { + "type": "object", + "properties": { + "reopened": { + "type": "number" + }, + "reverted": { + "type": "number" + }, + "superseded": { + "type": "number" + } + }, + "required": [ + "reopened", + "reverted", + "superseded" + ] + }, + "latestBacktestRun": { + "type": "object", + "nullable": true, + "properties": { + "corpusChecksum": { + "type": "string" + }, + "at": { + "type": "string" + } + }, + "required": [ + "corpusChecksum", + "at" + ] + } + }, + "required": [ + "windowDays", + "rules", + "reversals", + "latestBacktestRun" + ] } }, "required": [ @@ -583,6 +655,7 @@ "updatedAt", "totals", "weekly", + "rulePrecision", "byProject", "fleetAccuracy", "accuracyTrend", diff --git a/src/api/routes.ts b/src/api/routes.ts index 2d9853a6ef..5b8bcf8763 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -319,6 +319,7 @@ import { isFairnessAnalyticsEnabled, resolveFairnessAnalyticsManifestOverride } import { isRagEnabled } from "../review/rag-wire"; import { getPublicStats, isPublicStatsEnabled, resolvePublicStatsManifestOverride } from "../review/public-stats"; import { loadPublicAccuracyTrend } from "../services/public-accuracy-trend"; +import { loadPublicRulePrecision } from "../review/public-rule-precision"; import { loadCalibrationTrend } from "../services/rule-calibration-trend"; import { isSatisfactionFloorAutotuneEnabled, loadSatisfactionFloorStatus, runSatisfactionFloorLoosening } from "../services/satisfaction-floor-loosening-run"; import { loadLiveKnobStatuses } from "../services/knob-loosening-run"; @@ -1262,14 +1263,17 @@ export function createApp() { const publicStatsManifestOverride = await resolvePublicStatsManifestOverride(c.env); if (!isPublicStatsEnabled(c.env, publicStatsManifestOverride)) return c.json({ error: "not_found" }, 404); try { - const [stats, accuracyTrend, reuseRateTrend, reviewVolumeTrend] = await Promise.all([ + const [stats, accuracyTrend, reuseRateTrend, reviewVolumeTrend, rulePrecision] = await Promise.all([ getPublicStats(c.env), loadPublicAccuracyTrend(c.env), loadPublicReuseRateTrend(c.env), loadPublicReviewVolumeTrend(c.env), + // #8230: measured per-rule precision + the reproducibility freeze point. Same flag, same cache, + // same one-surface posture as the sibling trends. + loadPublicRulePrecision(c.env), ]); c.header("Cache-Control", "public, max-age=60, stale-while-revalidate=300"); - return c.json({ ...stats, accuracyTrend, reuseRateTrend, reviewVolumeTrend }); + return c.json({ ...stats, accuracyTrend, reuseRateTrend, reviewVolumeTrend, rulePrecision }); } catch { return c.json({ error: "public_stats_unavailable" }, 503); } diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 45b389ce94..8ed4a4dd89 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -111,6 +111,15 @@ export const PublicStatsSchema = z minutesSaved: z.number(), }), weekly: z.object({ reviewed: z.number(), merged: z.number() }), + /** Measured per-rule precision over the trailing window (#8230): decided human verdicts per rule with + * confirmed/decided precision, null below the public sample floor — plus all three reversal-shape + * counts and the latest backtest run's corpus checksum (the independently-verifiable freeze point). */ + rulePrecision: z.object({ + windowDays: z.number(), + rules: z.array(z.object({ ruleId: z.string(), decided: z.number(), precision: z.number().nullable() })), + reversals: z.object({ reopened: z.number(), reverted: z.number(), superseded: z.number() }), + latestBacktestRun: z.object({ corpusChecksum: z.string(), at: z.string() }).nullable(), + }), byProject: z.array( z.object({ project: z.string(), diff --git a/src/review/public-rule-precision.ts b/src/review/public-rule-precision.ts new file mode 100644 index 0000000000..0ae483c4e8 --- /dev/null +++ b/src/review/public-rule-precision.ts @@ -0,0 +1,101 @@ +// Public measured-accuracy surface (#8230, epic #8211 track G): per-rule precision over the trailing +// window, computed from the SAME human-verdict events the internal calibration reads — the public claim +// and the internal number can never diverge because they are one number. Aggregates and rule ids ONLY: +// no target keys, no repos, no confidence distributions, no corpus content (the issue's own exclusion +// list). Sparse rules report null precision, never a misreadable 0% — the same N/A-over-zero discipline +// as the #8085 scorer and the calibration trend. +// +// The `latestBacktestRun` block is the reproducibility hook (#8136's conclusion, operationalized): the +// most recent persisted backtest run's corpus checksum + timestamp is the freeze point a skeptic needs to +// independently re-run the comparison and verify the reported numbers are real. +import { safeAll } from "./public-stats"; + +/** Trailing window the public precision claim covers. Mirrors the 90-day corpus lookback the loosening + * loops evaluate over — the public number describes the same evidence the system acts on. */ +export const PUBLIC_PRECISION_WINDOW_DAYS = 90; + +/** Below this many decided cases a rule's precision is null on the public surface. Deliberately stiffer + * than the internal trend's weekly floor (MIN_CALIBRATION_TREND_SAMPLE = 3): a public percentage carries + * more weight than an operator chart, so it needs more evidence before it exists at all. */ +export const PUBLIC_PRECISION_MIN_DECIDED = 10; + +// Mirror signal-tracking-wire.ts's event-type folding (`signal.human_override:`) — the same local +// duplication rule-calibration-trend.ts documents for its identical queries. +const HUMAN_OVERRIDE_EVENT_TYPE_PREFIX = "signal.human_override:"; + +export type PublicRulePrecisionRow = { + ruleId: string; + decided: number; + /** confirmed / decided, rounded to 3 decimals; null below {@link PUBLIC_PRECISION_MIN_DECIDED}. */ + precision: number | null; +}; + +export type PublicRulePrecision = { + windowDays: number; + rules: PublicRulePrecisionRow[]; + /** All three reversal shapes counted over the window — the "counted against ourselves" number. */ + reversals: { reopened: number; reverted: number; superseded: number }; + /** The latest persisted backtest run carrying a corpus checksum — the independently-verifiable freeze + * point — or null when no run has been recorded yet. */ + latestBacktestRun: { corpusChecksum: string; at: string } | null; +}; + +/** + * Load the public per-rule precision block. Fail-safe per section (the same degradation contract as + * loadCalibrationTrend): a read error yields an empty/absent section, never a thrown public endpoint. + */ +export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now()): Promise { + const sinceIso = new Date(nowMs - PUBLIC_PRECISION_WINDOW_DAYS * 24 * 60 * 60 * 1000).toISOString(); + + const overrideRows = await safeAll<{ rule_id: string; decided: number; reversed: number }>( + env, + `SELECT substr(event_type, ${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX.length + 1}) AS rule_id, COUNT(*) AS decided, + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed + FROM audit_events + WHERE event_type LIKE '${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}%' AND created_at >= ? + GROUP BY rule_id`, + sinceIso, + ); + const rules: PublicRulePrecisionRow[] = overrideRows + .map((row) => { + /* v8 ignore next 2 -- SUM(CASE) over a GROUP BY always yields a defined integer; the ?? guards a + * future query-shape change, mirroring loadOverrideDayRows' identical note. */ + const reversed = row.reversed ?? 0; + const decided = row.decided; + return { + ruleId: row.rule_id, + decided, + precision: decided >= PUBLIC_PRECISION_MIN_DECIDED ? Math.round(((decided - reversed) / decided) * 1000) / 1000 : null, + }; + }) + .sort((a, b) => a.ruleId.localeCompare(b.ruleId)); + + const reversalRows = await safeAll<{ event_type: string; n: number }>( + env, + `SELECT event_type, COUNT(*) AS n FROM audit_events + WHERE event_type IN ('reversal_reopened', 'reversal_reverted', 'reversal_superseded') AND created_at >= ? + GROUP BY event_type`, + sinceIso, + ); + const reversalCount = (eventType: string) => reversalRows.find((row) => row.event_type === eventType)?.n ?? 0; + + const runRows = await safeAll<{ checksum: string; created_at: string }>( + env, + `SELECT json_extract(metadata_json, '$.corpusChecksum') AS checksum, created_at FROM audit_events + WHERE event_type IN ('calibration.threshold_backtest_run', 'calibration.logic_backtest_run') + AND json_extract(metadata_json, '$.corpusChecksum') IS NOT NULL + ORDER BY created_at DESC LIMIT 1`, + ); + const latest = runRows[0]; + + return { + windowDays: PUBLIC_PRECISION_WINDOW_DAYS, + rules, + reversals: { + reopened: reversalCount("reversal_reopened"), + reverted: reversalCount("reversal_reverted"), + superseded: reversalCount("reversal_superseded"), + }, + latestBacktestRun: latest && typeof latest.checksum === "string" && latest.checksum !== "" ? { corpusChecksum: latest.checksum, at: latest.created_at } : null, + }; +} diff --git a/test/integration/public-stats-route.test.ts b/test/integration/public-stats-route.test.ts index 372d63ffe4..8685728993 100644 --- a/test/integration/public-stats-route.test.ts +++ b/test/integration/public-stats-route.test.ts @@ -83,6 +83,15 @@ describe("GET /v1/public/stats (#1059)", () => { expect(res.status).toBe(200); expect(res.headers.get("cache-control")).toContain("max-age=60"); + // #8230: the measured-accuracy block rides the same surface, same flag, same cache. + const withPrecision = (await res.clone().json()) as { + rulePrecision: { windowDays: number; rules: unknown[]; reversals: Record; latestBacktestRun: unknown }; + }; + expect(withPrecision.rulePrecision.windowDays).toBe(90); + // seed() records one reversal_reopened — the block reads the same ledger the totals do. + expect(withPrecision.rulePrecision.reversals).toEqual({ reopened: 1, reverted: 0, superseded: 0 }); + expect(withPrecision.rulePrecision.latestBacktestRun).toBeNull(); + const body = (await res.json()) as { totals: Record; weekly: { reviewed: number; merged: number }; diff --git a/test/unit/public-rule-precision.test.ts b/test/unit/public-rule-precision.test.ts new file mode 100644 index 0000000000..84f552459b --- /dev/null +++ b/test/unit/public-rule-precision.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import { + loadPublicRulePrecision, + PUBLIC_PRECISION_MIN_DECIDED, + PUBLIC_PRECISION_WINDOW_DAYS, +} from "../../src/review/public-rule-precision"; +import { recordAuditEvent } from "../../src/db/repositories"; +import { createSignalStore } from "../../src/review/signal-tracking-wire"; +import { createTestEnv } from "../helpers/d1"; + +// #8230 (epic #8211 track G): the public measured-accuracy block. Load-bearing properties: the public +// number IS the internal number (same events), sparse rules are null (never a misreadable 0%), all three +// reversal shapes count, the reproducibility freeze point surfaces, and nothing target-identifying leaks. + +const NOW = Date.parse("2026-07-23T12:00:00.000Z"); + +async function seedVerdicts(env: Env, ruleId: string, confirmed: number, reversed: number): Promise { + const store = createSignalStore(env); + for (let i = 0; i < confirmed + reversed; i += 1) { + await store.recordHumanOverride({ + ruleId, + targetKey: `acme/widgets#${i + 1}`, + verdict: i < confirmed ? "confirmed" : "reversed", + occurredAt: new Date(NOW - 1000 - i).toISOString(), + }); + } +} + +describe("loadPublicRulePrecision (#8230)", () => { + it("computes per-rule precision from the same human-verdict events the internal calibration reads, sorted by rule id", async () => { + const env = createTestEnv(); + await seedVerdicts(env, "linked_issue_scope_mismatch", 9, 3); // 12 decided, precision 0.75 + await seedVerdicts(env, "ai_consensus_defect", 20, 5); // 25 decided, precision 0.8 + + const block = await loadPublicRulePrecision(env, NOW); + expect(block.windowDays).toBe(PUBLIC_PRECISION_WINDOW_DAYS); + expect(block.rules).toEqual([ + { ruleId: "ai_consensus_defect", decided: 25, precision: 0.8 }, + { ruleId: "linked_issue_scope_mismatch", decided: 12, precision: 0.75 }, + ]); + }); + + it("reports null precision below the public sample floor and excludes events outside the window", async () => { + const env = createTestEnv(); + await seedVerdicts(env, "sparse_rule", PUBLIC_PRECISION_MIN_DECIDED - 1, 0); // one short of the floor + // A decided verdict OUTSIDE the trailing window must not count toward anything. + await createSignalStore(env).recordHumanOverride({ + ruleId: "sparse_rule", + targetKey: "acme/widgets#999", + verdict: "confirmed", + occurredAt: new Date(NOW - (PUBLIC_PRECISION_WINDOW_DAYS + 5) * 24 * 60 * 60 * 1000).toISOString(), + }); + + const block = await loadPublicRulePrecision(env, NOW); + expect(block.rules).toEqual([{ ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null }]); + }); + + it("counts all three reversal shapes over the window and surfaces the latest backtest run's corpus checksum", async () => { + const env = createTestEnv(); + for (const [eventType, count] of [ + ["reversal_reopened", 2], + ["reversal_reverted", 1], + ["reversal_superseded", 3], + ] as const) { + for (let i = 0; i < count; i += 1) { + await recordAuditEvent(env, { eventType, targetKey: `acme/widgets#${i}`, outcome: "completed", createdAt: new Date(NOW - 5000 - i).toISOString() }); + } + } + // Two runs: the LATEST one's checksum must win; a run without a checksum (threshold-shaped metadata) + // must never be picked over an older one that carries it. + await recordAuditEvent(env, { + eventType: "calibration.logic_backtest_run", + targetKey: "rule", + outcome: "completed", + metadata: { corpusChecksum: "older000", comparison: {} }, + createdAt: new Date(NOW - 60_000).toISOString(), + }); + await recordAuditEvent(env, { + eventType: "calibration.logic_backtest_run", + targetKey: "rule", + outcome: "completed", + metadata: { corpusChecksum: "newest111", comparison: {} }, + createdAt: new Date(NOW - 30_000).toISOString(), + }); + await recordAuditEvent(env, { + eventType: "calibration.threshold_backtest_run", + targetKey: "rule", + outcome: "completed", + metadata: { comparison: {} }, // no checksum — filtered out by the query, not coerced + createdAt: new Date(NOW - 1000).toISOString(), + }); + + const block = await loadPublicRulePrecision(env, NOW); + expect(block.reversals).toEqual({ reopened: 2, reverted: 1, superseded: 3 }); + expect(block.latestBacktestRun).toEqual({ corpusChecksum: "newest111", at: new Date(NOW - 30_000).toISOString() }); + }); + + it("degrades fail-safe on a broken store and reports null freeze point on a fresh ledger", async () => { + const empty = await loadPublicRulePrecision(createTestEnv(), NOW); + expect(empty).toEqual({ + windowDays: PUBLIC_PRECISION_WINDOW_DAYS, + rules: [], + reversals: { reopened: 0, reverted: 0, superseded: 0 }, + latestBacktestRun: null, + }); + + const broken = createTestEnv(); + broken.DB = { prepare: () => { throw new Error("boom"); } } as never; + expect(await loadPublicRulePrecision(broken, NOW)).toEqual(empty); + }); + + it("INVARIANT: the public payload never carries target keys, repos, confidences, or private terms", async () => { + const env = createTestEnv(); + await seedVerdicts(env, "ai_consensus_defect", 15, 5); + await recordAuditEvent(env, { + eventType: "calibration.logic_backtest_run", + targetKey: "acme/widgets#7", + outcome: "completed", + metadata: { corpusChecksum: "abc123", comparison: {} }, + createdAt: new Date(NOW - 1000).toISOString(), + }); + const serialized = JSON.stringify(await loadPublicRulePrecision(env, NOW)); + expect(serialized).not.toMatch(/acme|#\d|targetKey|confidence|wallet|hotkey|trust|reward|payout/i); + }); +});