From c470097f1d112c2a5bccfa14dfe311161389f773 Mon Sep 17 00:00:00 2001 From: RealDiligent Date: Thu, 23 Jul 2026 05:55:05 +0800 Subject: [PATCH] feat(calibration): pure confusion-matrix scorer for candidate rule classifiers (#8085) --- .../src/calibration/backtest-score.ts | 65 +++++++++ packages/loopover-engine/src/index.ts | 1 + .../test/backtest-score.test.ts | 123 ++++++++++++++++++ test/unit/backtest-score-engine.test.ts | 113 ++++++++++++++++ 4 files changed, 302 insertions(+) create mode 100644 packages/loopover-engine/src/calibration/backtest-score.ts create mode 100644 packages/loopover-engine/test/backtest-score.test.ts create mode 100644 test/unit/backtest-score-engine.test.ts diff --git a/packages/loopover-engine/src/calibration/backtest-score.ts b/packages/loopover-engine/src/calibration/backtest-score.ts new file mode 100644 index 0000000000..9f08d01d52 --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-score.ts @@ -0,0 +1,65 @@ +// Backtest confusion-matrix scorer (#8085) -- replays a caller-supplied candidate classifier over a labeled +// BacktestCase corpus (#8083) and scores it against the real human verdicts, answering "if THIS version of +// the rule had been run against the same targets, would it have gotten more of them right?". Mirrors +// src/review/auto-tune.ts's GateEvalRow confusion-matrix shape (wouldMerge/mergeConfirmed/mergeFalse/ +// decided/mergePrecision), but at a backtest-replay grain instead of a live-eval grain. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestCase } from "./backtest-corpus.js"; + +// Convention: "reversed" is the positive class. A classifier that correctly predicts a case's real +// label of "reversed" (i.e. correctly identifies that the rule's original firing was WRONG) is a true +// positive. This is a deliberate, non-obvious choice — keep this comment attached to the type. +export type BacktestScoreReport = { + ruleId: string; + caseCount: number; + truePositive: number; + falsePositive: number; + trueNegative: number; + falseNegative: number; + precision: number | null; + recall: number | null; +}; + +/** + * Score `classify` against every case in `cases` carrying this `ruleId`, accumulating the four + * confusion-matrix counts against the real human labels ("reversed" is the positive class -- see the + * report type's own convention comment). Cases for a different `ruleId` are excluded from every count, + * `caseCount` included -- mirrors computeRulePrecision's (signal-tracking.ts) defensive override filter. + * `precision`/`recall` are null when their denominator is 0, never coerced to 0 or 1 -- the same "unknown + * stays unknown" discipline as RulePrecisionReport.precision. `classify` is deliberately synchronous: every + * case must be scorable without I/O, so a caller can replay thousands of historical cases against a fast, + * in-memory candidate rule implementation. + */ +export function scoreBacktest( + ruleId: string, + cases: readonly BacktestCase[], + classify: (backtestCase: BacktestCase) => "reversed" | "confirmed", +): BacktestScoreReport { + let caseCount = 0; + let truePositive = 0; + let falsePositive = 0; + let trueNegative = 0; + let falseNegative = 0; + for (const backtestCase of cases) { + if (backtestCase.ruleId !== ruleId) continue; + caseCount += 1; + const predicted = classify(backtestCase); + if (predicted === "reversed") { + if (backtestCase.label === "reversed") truePositive += 1; + else falsePositive += 1; + } else if (backtestCase.label === "confirmed") trueNegative += 1; + else falseNegative += 1; + } + return { + ruleId, + caseCount, + truePositive, + falsePositive, + trueNegative, + falseNegative, + precision: truePositive + falsePositive > 0 ? truePositive / (truePositive + falsePositive) : null, + recall: truePositive + falseNegative > 0 ? truePositive / (truePositive + falseNegative) : null, + }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 15c82cee67..6761786a77 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -164,6 +164,7 @@ export * from "./governor/action-mode.js"; export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; +export * from "./calibration/backtest-score.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-score.test.ts b/packages/loopover-engine/test/backtest-score.test.ts new file mode 100644 index 0000000000..1016bce72e --- /dev/null +++ b/packages/loopover-engine/test/backtest-score.test.ts @@ -0,0 +1,123 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { scoreBacktest, type BacktestCase } from "../dist/index.js"; + +function corpusCase(targetKey: string, label: BacktestCase["label"], overrides: Partial = {}): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey, + outcome: "block", + label, + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + ...overrides, + }; +} + +test("barrel: the public entrypoint re-exports the backtest scorer (#8085)", () => { + assert.equal(typeof scoreBacktest, "function"); +}); + +test("scoreBacktest: an all-correct classifier scores precision 1 and recall 1", () => { + const cases = [ + corpusCase("a#1", "reversed"), + corpusCase("a#2", "confirmed"), + corpusCase("a#3", "reversed"), + ]; + const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) => backtestCase.label); + assert.deepEqual(report, { + ruleId: "missing_linked_issue", + caseCount: 3, + truePositive: 2, + falsePositive: 0, + trueNegative: 1, + falseNegative: 0, + precision: 1, + recall: 1, + }); +}); + +test("scoreBacktest: an all-wrong classifier scores precision 0 and recall 0, with the misses in the right cells", () => { + const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed")]; + const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) => + backtestCase.label === "reversed" ? "confirmed" : "reversed", + ); + assert.deepEqual(report, { + ruleId: "missing_linked_issue", + caseCount: 2, + truePositive: 0, + falsePositive: 1, // predicted reversed on the confirmed-labeled case + trueNegative: 0, + falseNegative: 1, // predicted confirmed on the reversed-labeled case + precision: 0, + recall: 0, + }); +}); + +test("scoreBacktest: a mixed classifier accumulates all four confusion-matrix cells", () => { + const cases = [ + corpusCase("a#1", "reversed"), // predicted reversed -> truePositive + corpusCase("a#2", "confirmed"), // predicted reversed -> falsePositive + corpusCase("a#3", "confirmed"), // predicted confirmed -> trueNegative + corpusCase("a#4", "reversed"), // predicted confirmed -> falseNegative + ]; + const predictReversedFor = new Set(["a#1", "a#2"]); + const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) => + predictReversedFor.has(backtestCase.targetKey) ? "reversed" : "confirmed", + ); + assert.deepEqual(report, { + ruleId: "missing_linked_issue", + caseCount: 4, + truePositive: 1, + falsePositive: 1, + trueNegative: 1, + falseNegative: 1, + precision: 0.5, + recall: 0.5, + }); +}); + +test("scoreBacktest: an empty corpus reports zero counts with precision AND recall null", () => { + const report = scoreBacktest("missing_linked_issue", [], () => "reversed"); + assert.deepEqual(report, { + ruleId: "missing_linked_issue", + caseCount: 0, + truePositive: 0, + falsePositive: 0, + trueNegative: 0, + falseNegative: 0, + precision: null, + recall: null, + }); +}); + +test("scoreBacktest: precision is null (not 0) when the classifier never predicts reversed, while recall stays real", () => { + const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "reversed")], () => "confirmed"); + assert.equal(report.precision, null); // truePositive + falsePositive === 0 + assert.equal(report.recall, 0); // truePositive / (0 + 1 falseNegative) +}); + +test("scoreBacktest: recall is null (not 0) when no case is labeled reversed, while precision stays real", () => { + const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "confirmed")], () => "reversed"); + assert.equal(report.recall, null); // truePositive + falseNegative === 0 + assert.equal(report.precision, 0); // truePositive / (0 + 1 falsePositive) +}); + +test("scoreBacktest: cases for a different ruleId are excluded from every count, caseCount included", () => { + const report = scoreBacktest( + "missing_linked_issue", + [corpusCase("a#1", "reversed", { ruleId: "other_rule" }), corpusCase("a#2", "reversed")], + () => "reversed", + ); + assert.deepEqual(report, { + ruleId: "missing_linked_issue", + caseCount: 1, + truePositive: 1, + falsePositive: 0, + trueNegative: 0, + falseNegative: 0, + precision: 1, + recall: 1, + }); +}); diff --git a/test/unit/backtest-score-engine.test.ts b/test/unit/backtest-score-engine.test.ts new file mode 100644 index 0000000000..87e65cb3b8 --- /dev/null +++ b/test/unit/backtest-score-engine.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "vitest"; + +// Import the engine SOURCE directly (not the built dist) -- coverage.include lists +// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in +// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace +// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts. +import { scoreBacktest } from "../../packages/loopover-engine/src/calibration/backtest-score"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus"; + +function corpusCase(targetKey: string, label: BacktestCase["label"], overrides: Partial = {}): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey, + outcome: "block", + label, + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + ...overrides, + }; +} + +describe("scoreBacktest (#8085)", () => { + it("scores an all-correct classifier at precision 1 / recall 1", () => { + const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed"), corpusCase("a#3", "reversed")]; + expect(scoreBacktest("missing_linked_issue", cases, (backtestCase) => backtestCase.label)).toEqual({ + ruleId: "missing_linked_issue", + caseCount: 3, + truePositive: 2, + falsePositive: 0, + trueNegative: 1, + falseNegative: 0, + precision: 1, + recall: 1, + }); + }); + + it("scores an all-wrong classifier at precision 0 / recall 0 with the misses in the right cells", () => { + const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed")]; + expect( + scoreBacktest("missing_linked_issue", cases, (backtestCase) => + backtestCase.label === "reversed" ? "confirmed" : "reversed", + ), + ).toEqual({ + ruleId: "missing_linked_issue", + caseCount: 2, + truePositive: 0, + falsePositive: 1, + trueNegative: 0, + falseNegative: 1, + precision: 0, + recall: 0, + }); + }); + + it("accumulates all four confusion-matrix cells for a mixed classifier", () => { + const cases = [ + corpusCase("a#1", "reversed"), + corpusCase("a#2", "confirmed"), + corpusCase("a#3", "confirmed"), + corpusCase("a#4", "reversed"), + ]; + const predictReversedFor = new Set(["a#1", "a#2"]); + expect( + scoreBacktest("missing_linked_issue", cases, (backtestCase) => + predictReversedFor.has(backtestCase.targetKey) ? "reversed" : "confirmed", + ), + ).toEqual({ + ruleId: "missing_linked_issue", + caseCount: 4, + truePositive: 1, + falsePositive: 1, + trueNegative: 1, + falseNegative: 1, + precision: 0.5, + recall: 0.5, + }); + }); + + it("reports zero counts with precision AND recall null for an empty corpus", () => { + expect(scoreBacktest("missing_linked_issue", [], () => "reversed")).toEqual({ + ruleId: "missing_linked_issue", + caseCount: 0, + truePositive: 0, + falsePositive: 0, + trueNegative: 0, + falseNegative: 0, + precision: null, + recall: null, + }); + }); + + it("keeps precision null (not 0) when the classifier never predicts reversed, while recall stays real", () => { + const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "reversed")], () => "confirmed"); + expect(report.precision).toBeNull(); + expect(report.recall).toBe(0); + }); + + it("keeps recall null (not 0) when no case is labeled reversed, while precision stays real", () => { + const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "confirmed")], () => "reversed"); + expect(report.recall).toBeNull(); + expect(report.precision).toBe(0); + }); + + it("excludes cases for a different ruleId from every count, caseCount included", () => { + const report = scoreBacktest( + "missing_linked_issue", + [corpusCase("a#1", "reversed", { ruleId: "other_rule" }), corpusCase("a#2", "reversed")], + () => "reversed", + ); + expect(report.caseCount).toBe(1); + expect(report.truePositive).toBe(1); + }); +});