diff --git a/packages/loopover-engine/src/calibration/backtest-corpus.ts b/packages/loopover-engine/src/calibration/backtest-corpus.ts new file mode 100644 index 0000000000..5abc81412d --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-corpus.ts @@ -0,0 +1,96 @@ +// Labeled backtest corpus builder (#8083, part of the #8082 rule-precision backtest epic). Where +// signal-tracking.ts's computeRulePrecision collapses a rule's fired/override history into ONE aggregate +// precision number, a backtest needs each PAIRED case kept as an individual labeled record -- a concrete +// "this rule fired against this target, and a human later said it was right/wrong" row, replayable against +// a different candidate rule/classifier later. +// +// SELF-CONTAINED, STORAGE-AGNOSTIC: pure TypeScript, no IO, no DB, no env -- only the existing +// RuleFiredEvent/HumanOverrideEvent types from signal-tracking.ts, mirroring that module's own +// "pure calibration math here, storage at the host layer" posture (see its header comment). + +import type { HumanOverrideEvent, RuleFiredEvent } from "./signal-tracking.js"; + +/** One labeled backtest case: a specific rule firing (`firedAt`, with the fired event's `outcome` and + * optional `metadata`) joined to the human verdict that judged it (`label`, at `decidedAt`). `metadata` + * is omitted entirely (not set to `undefined`) when the fired event carried none -- the same + * optional-property discipline {@link RuleFiredEvent} itself uses. */ +export type BacktestCase = { + ruleId: string; + targetKey: string; + outcome: string; + label: "reversed" | "confirmed"; + firedAt: string; + decidedAt: string; + metadata?: Record; +}; + +/** True for an override event that targets the same rule as `ruleId` -- mirrors signal-tracking.ts's own + * private `overrideMatchesRule` helper (deliberately NOT exported from there; this issue is additive-only + * in a new file, so the one-line filter is mirrored here instead). */ +function overrideMatchesRule(event: HumanOverrideEvent, ruleId: string): boolean { + return event.ruleId === ruleId; +} + +/** + * Pick the override that judges `firedEvent`: the one whose `occurredAt` is closest in time strictly AFTER + * the fired event's own `occurredAt` (a verdict naturally follows the firing it judges); when none strictly + * follows it (e.g. clock skew, or a verdict recorded against an earlier fire of the same target), fall back + * to the most recent override by `occurredAt`. Ties keep the first candidate encountered, so the choice is + * deterministic for a fixed input order. + */ +function pairedOverrideFor(firedEvent: RuleFiredEvent, candidates: readonly HumanOverrideEvent[]): HumanOverrideEvent | null { + const firedMs = Date.parse(firedEvent.occurredAt); + let nearestAfter: HumanOverrideEvent | null = null; + let nearestAfterMs = Number.POSITIVE_INFINITY; + let mostRecent: HumanOverrideEvent | null = null; + let mostRecentMs = Number.NEGATIVE_INFINITY; + for (const candidate of candidates) { + const candidateMs = Date.parse(candidate.occurredAt); + if (candidateMs > firedMs && candidateMs - firedMs < nearestAfterMs) { + nearestAfter = candidate; + nearestAfterMs = candidateMs - firedMs; + } + if (candidateMs > mostRecentMs) { + mostRecent = candidate; + mostRecentMs = candidateMs; + } + } + return nearestAfter ?? mostRecent; +} + +/** + * Build the labeled backtest corpus for `ruleId` from its fired + override events. A fired event and an + * override pair into one {@link BacktestCase} when both carry the function's `ruleId` AND the same + * `targetKey`. A fired event with NO matching override is excluded from the result (not included as an + * unlabeled case) -- mirrors {@link computeRulePrecision}'s "only the decided ones count" discipline. + * When a target has been re-fired and re-judged more than once, each fired event pairs with the override + * whose `occurredAt` is closest in time strictly after that specific firing; if none strictly follows it, + * the most recent override by `occurredAt` stands in (see {@link pairedOverrideFor}) -- one case per fired + * event, never duplicates. Only events whose `ruleId` equals the argument are considered; like + * computeRulePrecision, a caller MAY pass mixed-rule event lists without filtering first. + */ +export function buildBacktestCorpus( + ruleId: string, + fired: readonly RuleFiredEvent[], + overrides: readonly HumanOverrideEvent[], +): BacktestCase[] { + const cases: BacktestCase[] = []; + for (const firedEvent of fired) { + if (firedEvent.ruleId !== ruleId) continue; + const candidates = overrides.filter( + (event) => overrideMatchesRule(event, ruleId) && event.targetKey === firedEvent.targetKey, + ); + const override = pairedOverrideFor(firedEvent, candidates); + if (!override) continue; + cases.push({ + ruleId, + targetKey: firedEvent.targetKey, + outcome: firedEvent.outcome, + label: override.verdict, + firedAt: firedEvent.occurredAt, + decidedAt: override.occurredAt, + ...(firedEvent.metadata !== undefined ? { metadata: firedEvent.metadata } : {}), + }); + } + return cases; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 609b2f2c10..15c82cee67 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -163,6 +163,7 @@ export * from "./governor/kill-switch.js"; export * from "./governor/action-mode.js"; export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; +export * from "./calibration/backtest-corpus.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-corpus.test.ts b/packages/loopover-engine/test/backtest-corpus.test.ts new file mode 100644 index 0000000000..784dfcdc05 --- /dev/null +++ b/packages/loopover-engine/test/backtest-corpus.test.ts @@ -0,0 +1,108 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "../dist/index.js"; + +function fired(ruleId: string, targetKey: string, overrides: Partial = {}): RuleFiredEvent { + return { ruleId, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides }; +} + +function override( + ruleId: string, + targetKey: string, + verdict: HumanOverrideEvent["verdict"], + overrides: Partial = {}, +): HumanOverrideEvent { + return { ruleId, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides }; +} + +test("barrel: the public entrypoint re-exports the backtest-corpus builder (#8083)", () => { + assert.equal(typeof buildBacktestCorpus, "function"); +}); + +test("buildBacktestCorpus: empty inputs produce an empty corpus", () => { + assert.deepEqual(buildBacktestCorpus("missing_linked_issue", [], []), []); +}); + +test("buildBacktestCorpus: a fired event with no matching override is excluded, not emitted unlabeled", () => { + const corpus = buildBacktestCorpus("missing_linked_issue", [fired("missing_linked_issue", "a#1")], []); + assert.deepEqual(corpus, []); +}); + +test("buildBacktestCorpus: a single fired+override pair produces one correctly-labeled case", () => { + const corpus = buildBacktestCorpus( + "missing_linked_issue", + [fired("missing_linked_issue", "a#1", { outcome: "closed", metadata: { pr: 1 } })], + [override("missing_linked_issue", "a#1", "reversed")], + ); + const expected: BacktestCase[] = [ + { + ruleId: "missing_linked_issue", + targetKey: "a#1", + outcome: "closed", + label: "reversed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + metadata: { pr: 1 }, + }, + ]; + assert.deepEqual(corpus, expected); +}); + +test("buildBacktestCorpus: omits the metadata key entirely when the fired event carries none", () => { + const [item] = buildBacktestCorpus( + "missing_linked_issue", + [fired("missing_linked_issue", "a#1")], + [override("missing_linked_issue", "a#1", "confirmed")], + ); + assert.ok(item); + assert.equal(item.label, "confirmed"); + assert.equal(Object.hasOwn(item, "metadata"), false); +}); + +test("buildBacktestCorpus: multiple overrides pair each firing with the nearest strictly-following verdict", () => { + const corpus = buildBacktestCorpus( + "rule", + [ + fired("rule", "a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), + fired("rule", "a#1", { occurredAt: "2026-07-22T04:00:00.000Z" }), + ], + [ + override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }), + override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T06:00:00.000Z" }), + ], + ); + assert.equal(corpus.length, 2); + assert.deepEqual( + corpus.map((item) => [item.firedAt, item.label, item.decidedAt]), + [ + ["2026-07-22T00:00:00.000Z", "reversed", "2026-07-22T02:00:00.000Z"], + ["2026-07-22T04:00:00.000Z", "confirmed", "2026-07-22T06:00:00.000Z"], + ], + ); +}); + +test("buildBacktestCorpus: with no strictly-following override, the most recent one stands in", () => { + const corpus = buildBacktestCorpus( + "rule", + [fired("rule", "a#1", { occurredAt: "2026-07-22T10:00:00.000Z" })], + [ + override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T05:00:00.000Z" }), + ], + ); + assert.equal(corpus.length, 1); + assert.equal(corpus[0]?.label, "confirmed"); + assert.equal(corpus[0]?.decidedAt, "2026-07-22T05:00:00.000Z"); +}); + +test("buildBacktestCorpus: fired and override events for a DIFFERENT ruleId are ignored entirely", () => { + const corpus = buildBacktestCorpus( + "rule", + [fired("other_rule", "a#1"), fired("rule", "a#2")], + [override("rule", "a#1", "confirmed"), override("other_rule", "a#2", "confirmed"), override("rule", "a#2", "reversed")], + ); + assert.equal(corpus.length, 1); + assert.equal(corpus[0]?.targetKey, "a#2"); + assert.equal(corpus[0]?.label, "reversed"); +}); diff --git a/test/unit/backtest-corpus.test.ts b/test/unit/backtest-corpus.test.ts new file mode 100644 index 0000000000..954a17b4b2 --- /dev/null +++ b/test/unit/backtest-corpus.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; +import { + buildBacktestCorpus, + type BacktestCase, +} from "../../packages/loopover-engine/src/calibration/backtest-corpus"; +import type { HumanOverrideEvent, RuleFiredEvent } from "../../packages/loopover-engine/src/calibration/signal-tracking"; + +function fired(ruleId: string, targetKey: string, overrides: Partial = {}): RuleFiredEvent { + return { ruleId, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides }; +} + +function override( + ruleId: string, + targetKey: string, + verdict: HumanOverrideEvent["verdict"], + overrides: Partial = {}, +): HumanOverrideEvent { + return { ruleId, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides }; +} + +describe("buildBacktestCorpus (#8083) — pure labeled-corpus builder", () => { + it("produces an empty corpus from empty inputs", () => { + expect(buildBacktestCorpus("missing_linked_issue", [], [])).toEqual([]); + }); + + it("excludes a fired event with no matching override instead of emitting an unlabeled case", () => { + expect(buildBacktestCorpus("missing_linked_issue", [fired("missing_linked_issue", "a#1")], [])).toEqual([]); + }); + + it("pairs a single fired+override into one correctly-labeled case, carrying outcome and metadata", () => { + const corpus = buildBacktestCorpus( + "missing_linked_issue", + [fired("missing_linked_issue", "a#1", { outcome: "closed", metadata: { pr: 1 } })], + [override("missing_linked_issue", "a#1", "reversed")], + ); + const expected: BacktestCase[] = [ + { + ruleId: "missing_linked_issue", + targetKey: "a#1", + outcome: "closed", + label: "reversed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + metadata: { pr: 1 }, + }, + ]; + expect(corpus).toEqual(expected); + }); + + it("omits the metadata key entirely (not undefined) when the fired event has none", () => { + const [item] = buildBacktestCorpus( + "missing_linked_issue", + [fired("missing_linked_issue", "a#1")], + [override("missing_linked_issue", "a#1", "confirmed")], + ); + expect(item?.label).toBe("confirmed"); + expect(Object.hasOwn(item ?? {}, "metadata")).toBe(false); + }); + + it("pairs each firing with the NEAREST strictly-following override when a target was re-fired and re-judged", () => { + // Candidate scan order deliberately puts the farther-after verdict first, so the nearer one must + // replace it, and a third even-farther verdict must NOT displace the nearest already found. + const corpus = buildBacktestCorpus( + "rule", + [fired("rule", "a#1", { occurredAt: "2026-07-22T00:00:00.000Z" })], + [ + override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T06:00:00.000Z" }), + override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T02:00:00.000Z" }), + override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T08:00:00.000Z" }), + ], + ); + expect(corpus).toHaveLength(1); + expect(corpus[0]?.label).toBe("confirmed"); + expect(corpus[0]?.decidedAt).toBe("2026-07-22T02:00:00.000Z"); + }); + + it("re-fired targets each get their own case, never a duplicate for the same fired event", () => { + const corpus = buildBacktestCorpus( + "rule", + [ + fired("rule", "a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), + fired("rule", "a#1", { occurredAt: "2026-07-22T04:00:00.000Z" }), + ], + [ + override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }), + override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T06:00:00.000Z" }), + ], + ); + expect(corpus.map((item) => [item.firedAt, item.label])).toEqual([ + ["2026-07-22T00:00:00.000Z", "reversed"], + ["2026-07-22T04:00:00.000Z", "confirmed"], + ]); + }); + + it("falls back to the most recent override when none strictly follows the firing", () => { + // Scan order is ascending here, so the later verdict must replace the earlier as most-recent. + const corpus = buildBacktestCorpus( + "rule", + [fired("rule", "a#1", { occurredAt: "2026-07-22T10:00:00.000Z" })], + [ + override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T05:00:00.000Z" }), + ], + ); + expect(corpus).toHaveLength(1); + expect(corpus[0]?.label).toBe("confirmed"); + expect(corpus[0]?.decidedAt).toBe("2026-07-22T05:00:00.000Z"); + }); + + it("ignores fired and override events for a different ruleId, and overrides for a different target", () => { + const corpus = buildBacktestCorpus( + "rule", + [fired("other_rule", "a#1"), fired("rule", "a#2")], + [ + override("rule", "a#1", "confirmed"), // right rule, wrong target -- never pairs with a#2 + override("other_rule", "a#2", "confirmed"), // wrong rule, right target -- filtered out + override("rule", "a#2", "reversed"), + ], + ); + expect(corpus).toHaveLength(1); + expect(corpus[0]?.targetKey).toBe("a#2"); + expect(corpus[0]?.label).toBe("reversed"); + }); +});