diff --git a/packages/loopover-engine/src/calibration/backtest-corpus.ts b/packages/loopover-engine/src/calibration/backtest-corpus.ts new file mode 100644 index 0000000000..3d726b4b98 --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-corpus.ts @@ -0,0 +1,68 @@ +// Labeled backtest corpus builder (#8083) -- turns the calibration module's raw fired/override event history +// into a list of concrete "this rule fired against this target, and a human later said it was right/wrong" +// cases, each replayable against a different candidate rule/classifier later (see the parent epic #8082). +// +// SELF-CONTAINED, PURE: no IO, no DB, no env, no wall-clock read, and no imports beyond the existing +// RuleFiredEvent/HumanOverrideEvent types from signal-tracking.ts -- the same storage-agnostic discipline +// that whole module follows. `Date.parse` on the events' own `occurredAt` strings is not a clock read; it is +// pure parsing of caller-supplied data, so the function stays deterministic. + +import type { HumanOverrideEvent, RuleFiredEvent } from "./signal-tracking.js"; + +/** One labeled backtest case: a single rule firing paired with the human verdict that later decided it. + * `outcome` is the firing's own `RuleFiredEvent.outcome`; `label` is the paired `HumanOverrideEvent.verdict` + * (`"reversed"` = the rule was wrong that time, `"confirmed"` = it was right); `firedAt`/`decidedAt` are the + * two events' `occurredAt`. `metadata` carries the firing's own metadata, omitted entirely (never set to + * `undefined`) when the firing has none -- the same optional-property discipline `RuleFiredEvent` uses. */ +export type BacktestCase = { + ruleId: string; + targetKey: string; + outcome: string; + label: "reversed" | "confirmed"; + firedAt: string; + decidedAt: string; + metadata?: Record; +}; + +/** + * Build a labeled {@link BacktestCase} corpus for `ruleId` from its fired + override events. Only events whose + * `ruleId` matches the argument are considered (mirrors `overrideMatchesRule` in signal-tracking.ts: + * `event.ruleId === ruleId`); a caller MAY pass a mixed-rule list without filtering first. + * + * A firing with no matching override (same rule AND same `targetKey`) is EXCLUDED, not emitted as an + * unlabeled case -- the same "only the decided ones count" discipline as {@link computeRulePrecision}. + * + * Pairing when a `targetKey` was fired + judged more than once: each firing takes the override whose + * `occurredAt` is the nearest one STRICTLY AFTER that firing; if no override strictly follows it, the most + * recent override by `occurredAt` is used. Each firing yields at most one case (no duplicates for one firing). + */ +export function buildBacktestCorpus( + ruleId: string, + fired: readonly RuleFiredEvent[], + overrides: readonly HumanOverrideEvent[], +): BacktestCase[] { + // Mirrors overrideMatchesRule's one-line filter (event.ruleId === ruleId) in signal-tracking.ts. + const ruleOverrides = overrides.filter((override) => override.ruleId === ruleId); + const cases: BacktestCase[] = []; + for (const firing of fired) { + if (firing.ruleId !== ruleId) continue; + const candidates = ruleOverrides.filter((override) => override.targetKey === firing.targetKey); + if (candidates.length === 0) continue; + const firedMs = Date.parse(firing.occurredAt); + // candidates ascending by time: the first one strictly after the firing is the nearest-following match; + // when none follows, sorted[last] is the most-recent override overall (the documented fallback). + const sorted = [...candidates].sort((a, b) => Date.parse(a.occurredAt) - Date.parse(b.occurredAt)); + const decided = sorted.find((override) => Date.parse(override.occurredAt) > firedMs) ?? sorted[sorted.length - 1]!; + const backtestCase: BacktestCase = { + ruleId, + targetKey: firing.targetKey, + outcome: firing.outcome, + label: decided.verdict, + firedAt: firing.occurredAt, + decidedAt: decided.occurredAt, + }; + if (firing.metadata !== undefined) backtestCase.metadata = firing.metadata; + cases.push(backtestCase); + } + return cases; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 609b2f2c10..15c82cee67 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -163,6 +163,7 @@ export * from "./governor/kill-switch.js"; export * from "./governor/action-mode.js"; export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; +export * from "./calibration/backtest-corpus.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-corpus.test.ts b/packages/loopover-engine/test/backtest-corpus.test.ts new file mode 100644 index 0000000000..614419ecdb --- /dev/null +++ b/packages/loopover-engine/test/backtest-corpus.test.ts @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "../dist/index.js"; + +const RULE = "missing_linked_issue"; + +function fired(targetKey: string, overrides: Partial = {}): RuleFiredEvent { + return { ruleId: RULE, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides }; +} + +function override( + targetKey: string, + verdict: HumanOverrideEvent["verdict"], + overrides: Partial = {}, +): HumanOverrideEvent { + return { ruleId: RULE, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides }; +} + +test("barrel: the public entrypoint re-exports buildBacktestCorpus (#8083)", () => { + assert.equal(typeof buildBacktestCorpus, "function"); +}); + +test("buildBacktestCorpus: a fired event with no matching override is excluded (only decided cases count)", () => { + const corpus = buildBacktestCorpus(RULE, [fired("a#1"), fired("a#2")], [override("a#1", "confirmed")]); + assert.equal(corpus.length, 1); + assert.equal(corpus[0]!.targetKey, "a#1"); +}); + +test("buildBacktestCorpus: a single fired+override pair produces one correctly-labeled case", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1", { outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", metadata: { pr: 1 } })], + [override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" })], + ); + assert.deepEqual(corpus, [ + { + ruleId: RULE, + targetKey: "a#1", + outcome: "block", + label: "reversed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T02:00:00.000Z", + metadata: { pr: 1 }, + } satisfies BacktestCase, + ]); +}); + +test("buildBacktestCorpus: metadata is omitted entirely (not undefined) when the fired event has none", () => { + const corpus = buildBacktestCorpus(RULE, [fired("a#1")], [override("a#1", "confirmed")]); + assert.equal("metadata" in corpus[0]!, false); +}); + +test("buildBacktestCorpus: multiple overrides -> the firing pairs with the nearest override strictly after it", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" })], + [ + override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }), + override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("a#1", "confirmed", { occurredAt: "2026-07-21T23:00:00.000Z" }), + ], + ); + // The 01:00 override is the nearest one strictly after the 00:00 firing -> label "reversed". + assert.equal(corpus.length, 1); + assert.equal(corpus[0]!.label, "reversed"); + assert.equal(corpus[0]!.decidedAt, "2026-07-22T01:00:00.000Z"); +}); + +test("buildBacktestCorpus: when no override strictly follows the firing, the most recent override is used", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1", { occurredAt: "2026-07-22T05:00:00.000Z" })], + [ + override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }), + override("a#1", "confirmed", { occurredAt: "2026-07-22T04:00:00.000Z" }), + ], + ); + // Both overrides precede the 05:00 firing -> fall back to the most recent (04:00, "confirmed"). + assert.equal(corpus.length, 1); + assert.equal(corpus[0]!.label, "confirmed"); + assert.equal(corpus[0]!.decidedAt, "2026-07-22T04:00:00.000Z"); +}); + +test("buildBacktestCorpus: two firings for the same target each yield their own case (no duplicate for one firing)", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), fired("a#1", { occurredAt: "2026-07-22T02:30:00.000Z" })], + [ + override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }), + ], + ); + assert.equal(corpus.length, 2); + assert.deepEqual(corpus.map((c) => c.decidedAt), ["2026-07-22T01:00:00.000Z", "2026-07-22T03:00:00.000Z"]); +}); + +test("buildBacktestCorpus: fired and override events for a different ruleId are ignored", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1"), { ruleId: "other_rule", targetKey: "a#1", outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z" }], + [override("a#1", "confirmed"), { ruleId: "other_rule", targetKey: "a#1", verdict: "reversed", occurredAt: "2026-07-22T01:00:00.000Z" }], + ); + assert.equal(corpus.length, 1); + assert.equal(corpus[0]!.ruleId, RULE); + assert.equal(corpus[0]!.label, "confirmed"); +}); + +test("buildBacktestCorpus: empty input arrays produce an empty corpus", () => { + assert.deepEqual(buildBacktestCorpus(RULE, [], []), []); +}); diff --git a/test/unit/backtest-corpus.test.ts b/test/unit/backtest-corpus.test.ts new file mode 100644 index 0000000000..76a3666c71 --- /dev/null +++ b/test/unit/backtest-corpus.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "vitest"; +// Direct src-path import (not the `@loopover/engine` package barrel, which resolves to dist and is NOT in +// vitest's coverage.include): the engine's own node:test suite runs against dist and is invisible to Codecov +// (only review-enrichment has a c8 dist-remap harvest step; the engine has none), so this vitest test is what +// gives packages/loopover-engine/src/calibration/backtest-corpus.ts its codecov/patch coverage. The companion +// packages/loopover-engine/test/backtest-corpus.test.ts is the issue-required node:test that gates the engine +// workspace's own `npm run test`. Vite resolves the `.js` specifier to the sibling `.ts` on disk. +import { buildBacktestCorpus } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js"; +import type { HumanOverrideEvent, RuleFiredEvent } from "../../packages/loopover-engine/src/calibration/signal-tracking.js"; + +const RULE = "missing_linked_issue"; + +function fired(targetKey: string, overrides: Partial = {}): RuleFiredEvent { + return { ruleId: RULE, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides }; +} + +function override(targetKey: string, verdict: HumanOverrideEvent["verdict"], overrides: Partial = {}): HumanOverrideEvent { + return { ruleId: RULE, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides }; +} + +describe("buildBacktestCorpus (#8083)", () => { + it("excludes a fired event with no matching override (only decided cases count)", () => { + const corpus = buildBacktestCorpus(RULE, [fired("a#1"), fired("a#2")], [override("a#1", "confirmed")]); + expect(corpus.map((c) => c.targetKey)).toEqual(["a#1"]); + }); + + it("produces one correctly-labeled case for a single fired+override pair, carrying metadata through", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z", metadata: { pr: 1 } })], + [override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" })], + ); + expect(corpus).toEqual([ + { + ruleId: RULE, + targetKey: "a#1", + outcome: "block", + label: "reversed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T02:00:00.000Z", + metadata: { pr: 1 }, + } satisfies BacktestCase, + ]); + }); + + it("omits metadata entirely (never sets it to undefined) when the fired event has none", () => { + const corpus = buildBacktestCorpus(RULE, [fired("a#1")], [override("a#1", "confirmed")]); + expect("metadata" in corpus[0]!).toBe(false); + }); + + it("pairs a firing with the nearest override strictly after it when a target was judged multiple times", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" })], + [ + override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }), + override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("a#1", "confirmed", { occurredAt: "2026-07-21T23:00:00.000Z" }), + ], + ); + expect(corpus).toHaveLength(1); + expect(corpus[0]!.label).toBe("reversed"); + expect(corpus[0]!.decidedAt).toBe("2026-07-22T01:00:00.000Z"); + }); + + it("falls back to the most recent override when none strictly follows the firing", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1", { occurredAt: "2026-07-22T05:00:00.000Z" })], + [ + override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }), + override("a#1", "confirmed", { occurredAt: "2026-07-22T04:00:00.000Z" }), + ], + ); + expect(corpus[0]!.label).toBe("confirmed"); + expect(corpus[0]!.decidedAt).toBe("2026-07-22T04:00:00.000Z"); + }); + + it("gives each of two firings for the same target its own case (no duplicate for one firing)", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), fired("a#1", { occurredAt: "2026-07-22T02:30:00.000Z" })], + [ + override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }), + override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }), + ], + ); + expect(corpus.map((c) => c.decidedAt)).toEqual(["2026-07-22T01:00:00.000Z", "2026-07-22T03:00:00.000Z"]); + }); + + it("ignores fired and override events for a different ruleId", () => { + const corpus = buildBacktestCorpus( + RULE, + [fired("a#1"), { ruleId: "other_rule", targetKey: "a#1", outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z" }], + [override("a#1", "confirmed"), { ruleId: "other_rule", targetKey: "a#1", verdict: "reversed", occurredAt: "2026-07-22T01:00:00.000Z" }], + ); + expect(corpus).toHaveLength(1); + expect(corpus[0]!).toMatchObject({ ruleId: RULE, label: "confirmed" }); + }); + + it("returns an empty corpus for empty input arrays", () => { + expect(buildBacktestCorpus(RULE, [], [])).toEqual([]); + }); +});