diff --git a/packages/loopover-engine/src/calibration/backtest-corpus.ts b/packages/loopover-engine/src/calibration/backtest-corpus.ts new file mode 100644 index 0000000000..de7df80dcf --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-corpus.ts @@ -0,0 +1,88 @@ +// Labeled backtest corpus builder (#8083, parent epic #8082) -- turns the calibration module's raw +// RuleFiredEvent/HumanOverrideEvent history into the per-case labeled records a backtest needs. Where +// computeRulePrecision (signal-tracking.ts) aggregates the SAME pairing into one precision number, this +// keeps each paired firing as an individual replayable case: "this rule fired against this target, and a +// human later said it was right/wrong." +// +// SELF-CONTAINED AND PURE, like everything in this module: no IO, no DB, no env -- only the existing +// event types from signal-tracking.ts. Additive-only; no existing consumer changes behavior. + +import type { HumanOverrideEvent, RuleFiredEvent } from "./signal-tracking.js"; + +/** One labeled, replayable backtest case: a specific rule firing plus the human verdict it eventually got. + * `outcome` is the fired event's outcome; `label` is the override's verdict; `firedAt`/`decidedAt` carry the + * two events' own timestamps. `metadata` is the FIRED event's metadata, omitted entirely (not set to + * `undefined`) when the fired event has none -- the same optional-property discipline RuleFiredEvent uses. */ +export type BacktestCase = { + ruleId: string; + targetKey: string; + outcome: string; + label: "reversed" | "confirmed"; + firedAt: string; + decidedAt: string; + metadata?: Record; +}; + +/** + * Build the labeled corpus for `ruleId` from its fired + override events. A fired event pairs with an + * override when both carry the function's `ruleId` AND the same `targetKey`; a fired event with no matching + * override is EXCLUDED (not emitted as an unlabeled case) -- mirrors computeRulePrecision's "only the + * decided ones count" discipline (see that function's own doc comment in signal-tracking.ts). + * + * Pairing rule when a target has MULTIPLE overrides for the same rule (re-fired and re-judged more than + * once): each fired event pairs with the override whose `occurredAt` is closest in time strictly AFTER that + * specific fired event's `occurredAt`; when none strictly follows it, it falls back to the most recent + * override by `occurredAt`. Each fired event produces at most one BacktestCase -- never duplicates. + */ +export function buildBacktestCorpus( + ruleId: string, + fired: readonly RuleFiredEvent[], + overrides: readonly HumanOverrideEvent[], +): BacktestCase[] { + // Mirrors overrideMatchesRule's one-line filter in signal-tracking.ts (kept private there; this module is + // additive-only and must not modify that file to export it). + const matchingOverrides = overrides.filter((event) => event.ruleId === ruleId); + + const cases: BacktestCase[] = []; + for (const event of fired) { + if (event.ruleId !== ruleId) continue; + const candidates = matchingOverrides.filter((override) => override.targetKey === event.targetKey); + if (candidates.length === 0) continue; + + const firedAtMs = Date.parse(event.occurredAt); + let paired: HumanOverrideEvent | undefined; + let pairedDeltaMs = Number.POSITIVE_INFINITY; + for (const override of candidates) { + const deltaMs = Date.parse(override.occurredAt) - firedAtMs; + if (deltaMs > 0 && deltaMs < pairedDeltaMs) { + paired = override; + pairedDeltaMs = deltaMs; + } + } + if (!paired) { + // No override strictly follows this firing -- fall back to the most recent override by occurredAt. + let latestMs = Number.NEGATIVE_INFINITY; + for (const override of candidates) { + const occurredMs = Date.parse(override.occurredAt); + if (occurredMs > latestMs) { + paired = override; + latestMs = occurredMs; + } + } + } + + // `paired` is always set here: candidates is non-empty and the fallback scans every candidate with a + // strictly-greater-than comparison against -Infinity. + const pairedOverride = paired as HumanOverrideEvent; + cases.push({ + ruleId: event.ruleId, + targetKey: event.targetKey, + outcome: event.outcome, + label: pairedOverride.verdict, + firedAt: event.occurredAt, + decidedAt: pairedOverride.occurredAt, + ...(event.metadata !== undefined ? { metadata: event.metadata } : {}), + }); + } + return cases; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 609b2f2c10..15c82cee67 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -163,6 +163,7 @@ export * from "./governor/kill-switch.js"; export * from "./governor/action-mode.js"; export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; +export * from "./calibration/backtest-corpus.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-corpus.test.ts b/packages/loopover-engine/test/backtest-corpus.test.ts new file mode 100644 index 0000000000..6a09c32f05 --- /dev/null +++ b/packages/loopover-engine/test/backtest-corpus.test.ts @@ -0,0 +1,135 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "../dist/index.js"; + +// #8083: buildBacktestCorpus pairs each rule firing with its human verdict into a labeled, replayable +// BacktestCase. Mirrors signal-tracking.test.ts's fixture style; the pairing rule under test is the +// nearest-strictly-following override, falling back to the most recent when none follows. + +function fired(ruleId: string, targetKey: string, overrides: Partial = {}): RuleFiredEvent { + return { ruleId, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides }; +} + +function override( + ruleId: string, + targetKey: string, + verdict: HumanOverrideEvent["verdict"], + overrides: Partial = {}, +): HumanOverrideEvent { + return { ruleId, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides }; +} + +test("barrel: the public entrypoint re-exports the backtest-corpus builder (#8083)", () => { + assert.equal(typeof buildBacktestCorpus, "function"); +}); + +test("a single fired+override pair produces one correctly-labeled case", () => { + const corpus = buildBacktestCorpus( + "missing_linked_issue", + [fired("missing_linked_issue", "a/r#1", { metadata: { headSha: "abc" } })], + [override("missing_linked_issue", "a/r#1", "reversed")], + ); + assert.deepEqual(corpus, [ + { + ruleId: "missing_linked_issue", + targetKey: "a/r#1", + outcome: "block", + label: "reversed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + metadata: { headSha: "abc" }, + } satisfies BacktestCase, + ]); +}); + +test("a fired event with no matching override is excluded, not emitted unlabeled", () => { + const corpus = buildBacktestCorpus( + "missing_linked_issue", + [fired("missing_linked_issue", "a/r#1"), fired("missing_linked_issue", "a/r#2")], + [override("missing_linked_issue", "a/r#2", "confirmed")], + ); + assert.deepEqual( + corpus.map((backtestCase) => backtestCase.targetKey), + ["a/r#2"], + ); +}); + +test("metadata is omitted entirely (not set to undefined) when the fired event has none", () => { + const [backtestCase] = buildBacktestCorpus( + "missing_linked_issue", + [fired("missing_linked_issue", "a/r#1")], + [override("missing_linked_issue", "a/r#1", "confirmed")], + ); + assert.equal(Object.hasOwn(backtestCase!, "metadata"), false); +}); + +test("multiple overrides for one target pair each firing with the nearest strictly-following override", () => { + const corpus = buildBacktestCorpus( + "rule", + [ + fired("rule", "a/r#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), + fired("rule", "a/r#1", { occurredAt: "2026-07-22T02:00:00.000Z" }), + ], + [ + override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }), + ], + ); + assert.deepEqual( + corpus.map((backtestCase) => [backtestCase.label, backtestCase.decidedAt]), + [ + ["reversed", "2026-07-22T01:00:00.000Z"], + ["confirmed", "2026-07-22T03:00:00.000Z"], + ], + ); +}); + +test("a firing with no strictly-following override falls back to the most recent override", () => { + const corpus = buildBacktestCorpus( + "rule", + [fired("rule", "a/r#1", { occurredAt: "2026-07-22T05:00:00.000Z" })], + [ + override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }), + ], + ); + assert.deepEqual( + corpus.map((backtestCase) => [backtestCase.label, backtestCase.decidedAt]), + [["confirmed", "2026-07-22T03:00:00.000Z"]], + ); +}); + +test("an override at exactly the firing's own instant does not count as strictly following", () => { + const corpus = buildBacktestCorpus( + "rule", + [fired("rule", "a/r#1", { occurredAt: "2026-07-22T02:00:00.000Z" })], + [ + override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T02:00:00.000Z" }), + override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + ], + ); + // Neither override strictly follows, so the most recent one (02:00, "confirmed") wins the fallback. + assert.deepEqual( + corpus.map((backtestCase) => [backtestCase.label, backtestCase.decidedAt]), + [["confirmed", "2026-07-22T02:00:00.000Z"]], + ); +}); + +test("events for a different ruleId are ignored on both sides", () => { + const corpus = buildBacktestCorpus( + "rule", + [fired("other_rule", "a/r#1"), fired("rule", "a/r#2")], + [override("rule", "a/r#1", "reversed"), override("other_rule", "a/r#2", "reversed"), override("rule", "a/r#2", "confirmed")], + ); + assert.deepEqual( + corpus.map((backtestCase) => [backtestCase.targetKey, backtestCase.label]), + [["a/r#2", "confirmed"]], + ); +}); + +test("empty input arrays produce an empty corpus", () => { + assert.deepEqual(buildBacktestCorpus("rule", [], []), []); + assert.deepEqual(buildBacktestCorpus("rule", [fired("rule", "a/r#1")], []), []); + assert.deepEqual(buildBacktestCorpus("rule", [], [override("rule", "a/r#1", "reversed")]), []); +}); diff --git a/test/unit/backtest-corpus.test.ts b/test/unit/backtest-corpus.test.ts new file mode 100644 index 0000000000..0c6fcc8d77 --- /dev/null +++ b/test/unit/backtest-corpus.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from "vitest"; +import { + buildBacktestCorpus, + type BacktestCase, +} from "../../packages/loopover-engine/src/calibration/backtest-corpus.js"; +import type { HumanOverrideEvent, RuleFiredEvent } from "../../packages/loopover-engine/src/calibration/signal-tracking.js"; + +// #8083: root-side coverage twin of packages/loopover-engine/test/backtest-corpus.test.ts. The engine +// package's own node:test suite runs against dist/ (not instrumented by the root vitest coverage that +// Codecov gates on), so this file exercises the SAME contract directly against the engine src — the same +// direct-src import pattern test/contract/*-parity.test.ts and test/integration/miner-*.test.ts already use. + +function fired(ruleId: string, targetKey: string, overrides: Partial = {}): RuleFiredEvent { + return { ruleId, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides }; +} + +function override( + ruleId: string, + targetKey: string, + verdict: HumanOverrideEvent["verdict"], + overrides: Partial = {}, +): HumanOverrideEvent { + return { ruleId, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides }; +} + +describe("buildBacktestCorpus (#8083)", () => { + it("pairs a single fired+override into one labeled case, carrying outcome, timestamps, and metadata", () => { + const corpus = buildBacktestCorpus( + "missing_linked_issue", + [fired("missing_linked_issue", "a/r#1", { outcome: "exclude", metadata: { headSha: "abc" } })], + [override("missing_linked_issue", "a/r#1", "reversed")], + ); + expect(corpus).toEqual([ + { + ruleId: "missing_linked_issue", + targetKey: "a/r#1", + outcome: "exclude", + label: "reversed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + metadata: { headSha: "abc" }, + } satisfies BacktestCase, + ]); + }); + + it("excludes fired events with no matching override — only the decided ones count", () => { + const corpus = buildBacktestCorpus( + "rule", + [fired("rule", "a/r#1"), fired("rule", "a/r#2")], + [override("rule", "a/r#2", "confirmed")], + ); + expect(corpus.map((backtestCase) => backtestCase.targetKey)).toEqual(["a/r#2"]); + }); + + it("omits the metadata key entirely when the fired event has none", () => { + const [backtestCase] = buildBacktestCorpus( + "rule", + [fired("rule", "a/r#1")], + [override("rule", "a/r#1", "confirmed")], + ); + expect(Object.hasOwn(backtestCase!, "metadata")).toBe(false); + }); + + it("pairs each firing with the nearest strictly-following override when a target was judged repeatedly", () => { + const corpus = buildBacktestCorpus( + "rule", + [ + fired("rule", "a/r#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), + fired("rule", "a/r#1", { occurredAt: "2026-07-22T02:00:00.000Z" }), + ], + [ + override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }), + ], + ); + expect(corpus.map((backtestCase) => [backtestCase.label, backtestCase.decidedAt])).toEqual([ + ["reversed", "2026-07-22T01:00:00.000Z"], + ["confirmed", "2026-07-22T03:00:00.000Z"], + ]); + }); + + it("falls back to the most recent override when none strictly follows the firing (equal instants included)", () => { + const late = buildBacktestCorpus( + "rule", + [fired("rule", "a/r#1", { occurredAt: "2026-07-22T05:00:00.000Z" })], + [ + override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }), + ], + ); + expect(late.map((backtestCase) => backtestCase.decidedAt)).toEqual(["2026-07-22T03:00:00.000Z"]); + + const equalInstant = buildBacktestCorpus( + "rule", + [fired("rule", "a/r#1", { occurredAt: "2026-07-22T02:00:00.000Z" })], + [ + override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T02:00:00.000Z" }), + override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + ], + ); + expect(equalInstant.map((backtestCase) => backtestCase.label)).toEqual(["confirmed"]); + }); + + it("ignores events for a different ruleId on both the fired and override sides", () => { + const corpus = buildBacktestCorpus( + "rule", + [fired("other_rule", "a/r#1"), fired("rule", "a/r#2")], + [override("rule", "a/r#1", "reversed"), override("other_rule", "a/r#2", "reversed"), override("rule", "a/r#2", "confirmed")], + ); + expect(corpus.map((backtestCase) => [backtestCase.targetKey, backtestCase.label])).toEqual([["a/r#2", "confirmed"]]); + }); + + it("produces an empty corpus for empty inputs in every combination", () => { + expect(buildBacktestCorpus("rule", [], [])).toEqual([]); + expect(buildBacktestCorpus("rule", [fired("rule", "a/r#1")], [])).toEqual([]); + expect(buildBacktestCorpus("rule", [], [override("rule", "a/r#1", "reversed")])).toEqual([]); + }); +});