feat(calibration): pure BacktestCase corpus builder from fired/override events

packages/loopover-engine/src/calibration/backtest-corpus.ts

@@ -0,0 +1,68 @@
+// Labeled backtest corpus builder (#8083) -- turns the calibration module's raw fired/override event history
+// into a list of concrete "this rule fired against this target, and a human later said it was right/wrong"
+// cases, each replayable against a different candidate rule/classifier later (see the parent epic #8082).
+//
+// SELF-CONTAINED, PURE: no IO, no DB, no env, no wall-clock read, and no imports beyond the existing
+// RuleFiredEvent/HumanOverrideEvent types from signal-tracking.ts -- the same storage-agnostic discipline
+// that whole module follows. `Date.parse` on the events' own `occurredAt` strings is not a clock read; it is
+// pure parsing of caller-supplied data, so the function stays deterministic.
+
+import type { HumanOverrideEvent, RuleFiredEvent } from "./signal-tracking.js";
+
+/** One labeled backtest case: a single rule firing paired with the human verdict that later decided it.
+ *  `outcome` is the firing's own `RuleFiredEvent.outcome`; `label` is the paired `HumanOverrideEvent.verdict`
+ *  (`"reversed"` = the rule was wrong that time, `"confirmed"` = it was right); `firedAt`/`decidedAt` are the
+ *  two events' `occurredAt`. `metadata` carries the firing's own metadata, omitted entirely (never set to
+ *  `undefined`) when the firing has none -- the same optional-property discipline `RuleFiredEvent` uses. */
+export type BacktestCase = {
+  ruleId: string;
+  targetKey: string;
+  outcome: string;
+  label: "reversed" | "confirmed";
+  firedAt: string;
+  decidedAt: string;
+  metadata?: Record<string, unknown>;
+};
+
+/**
+ * Build a labeled {@link BacktestCase} corpus for `ruleId` from its fired + override events. Only events whose
+ * `ruleId` matches the argument are considered (mirrors `overrideMatchesRule` in signal-tracking.ts:
+ * `event.ruleId === ruleId`); a caller MAY pass a mixed-rule list without filtering first.
+ *
+ * A firing with no matching override (same rule AND same `targetKey`) is EXCLUDED, not emitted as an
+ * unlabeled case -- the same "only the decided ones count" discipline as {@link computeRulePrecision}.
+ *
+ * Pairing when a `targetKey` was fired + judged more than once: each firing takes the override whose
+ * `occurredAt` is the nearest one STRICTLY AFTER that firing; if no override strictly follows it, the most
+ * recent override by `occurredAt` is used. Each firing yields at most one case (no duplicates for one firing).
+ */
+export function buildBacktestCorpus(
+  ruleId: string,
+  fired: readonly RuleFiredEvent[],
+  overrides: readonly HumanOverrideEvent[],
+): BacktestCase[] {
+  // Mirrors overrideMatchesRule's one-line filter (event.ruleId === ruleId) in signal-tracking.ts.
+  const ruleOverrides = overrides.filter((override) => override.ruleId === ruleId);
+  const cases: BacktestCase[] = [];
+  for (const firing of fired) {
+    if (firing.ruleId !== ruleId) continue;
+    const candidates = ruleOverrides.filter((override) => override.targetKey === firing.targetKey);
+    if (candidates.length === 0) continue;
+    const firedMs = Date.parse(firing.occurredAt);
+    // candidates ascending by time: the first one strictly after the firing is the nearest-following match;
+    // when none follows, sorted[last] is the most-recent override overall (the documented fallback).
+    const sorted = [...candidates].sort((a, b) => Date.parse(a.occurredAt) - Date.parse(b.occurredAt));
+    const decided = sorted.find((override) => Date.parse(override.occurredAt) > firedMs) ?? sorted[sorted.length - 1]!;
+    const backtestCase: BacktestCase = {
+      ruleId,
+      targetKey: firing.targetKey,
+      outcome: firing.outcome,
+      label: decided.verdict,
+      firedAt: firing.occurredAt,
+      decidedAt: decided.occurredAt,
+    };
+    if (firing.metadata !== undefined) backtestCase.metadata = firing.metadata;
+    cases.push(backtestCase);
+  }
+  return cases;
+}

packages/loopover-engine/src/index.ts

@@ -163,6 +163,7 @@ export * from "./governor/kill-switch.js";
 export * from "./governor/action-mode.js";
 export * from "./governor/chokepoint.js";
 export * from "./calibration/signal-tracking.js";
+export * from "./calibration/backtest-corpus.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/backtest-corpus.test.ts

@@ -0,0 +1,111 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "../dist/index.js";
+
+const RULE = "missing_linked_issue";
+
+function fired(targetKey: string, overrides: Partial<RuleFiredEvent> = {}): RuleFiredEvent {
+  return { ruleId: RULE, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides };
+}
+
+function override(
+  targetKey: string,
+  verdict: HumanOverrideEvent["verdict"],
+  overrides: Partial<HumanOverrideEvent> = {},
+): HumanOverrideEvent {
+  return { ruleId: RULE, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides };
+}
+
+test("barrel: the public entrypoint re-exports buildBacktestCorpus (#8083)", () => {
+  assert.equal(typeof buildBacktestCorpus, "function");
+});
+
+test("buildBacktestCorpus: a fired event with no matching override is excluded (only decided cases count)", () => {
+  const corpus = buildBacktestCorpus(RULE, [fired("a#1"), fired("a#2")], [override("a#1", "confirmed")]);
+  assert.equal(corpus.length, 1);
+  assert.equal(corpus[0]!.targetKey, "a#1");
+});
+
+test("buildBacktestCorpus: a single fired+override pair produces one correctly-labeled case", () => {
+  const corpus = buildBacktestCorpus(
+    RULE,
+    [fired("a#1", { outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", metadata: { pr: 1 } })],
+    [override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" })],
+  );
+  assert.deepEqual(corpus, [
+    {
+      ruleId: RULE,
+      targetKey: "a#1",
+      outcome: "block",
+      label: "reversed",
+      firedAt: "2026-07-22T00:00:00.000Z",
+      decidedAt: "2026-07-22T02:00:00.000Z",
+      metadata: { pr: 1 },
+    } satisfies BacktestCase,
+  ]);
+});
+
+test("buildBacktestCorpus: metadata is omitted entirely (not undefined) when the fired event has none", () => {
+  const corpus = buildBacktestCorpus(RULE, [fired("a#1")], [override("a#1", "confirmed")]);
+  assert.equal("metadata" in corpus[0]!, false);
+});
+
+test("buildBacktestCorpus: multiple overrides -> the firing pairs with the nearest override strictly after it", () => {
+  const corpus = buildBacktestCorpus(
+    RULE,
+    [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" })],
+    [
+      override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
+      override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+      override("a#1", "confirmed", { occurredAt: "2026-07-21T23:00:00.000Z" }),
+    ],
+  );
+  // The 01:00 override is the nearest one strictly after the 00:00 firing -> label "reversed".
+  assert.equal(corpus.length, 1);
+  assert.equal(corpus[0]!.label, "reversed");
+  assert.equal(corpus[0]!.decidedAt, "2026-07-22T01:00:00.000Z");
+});
+
+test("buildBacktestCorpus: when no override strictly follows the firing, the most recent override is used", () => {
+  const corpus = buildBacktestCorpus(
+    RULE,
+    [fired("a#1", { occurredAt: "2026-07-22T05:00:00.000Z" })],
+    [
+      override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+      override("a#1", "confirmed", { occurredAt: "2026-07-22T04:00:00.000Z" }),
+    ],
+  );
+  // Both overrides precede the 05:00 firing -> fall back to the most recent (04:00, "confirmed").
+  assert.equal(corpus.length, 1);
+  assert.equal(corpus[0]!.label, "confirmed");
+  assert.equal(corpus[0]!.decidedAt, "2026-07-22T04:00:00.000Z");
+});
+
+test("buildBacktestCorpus: two firings for the same target each yield their own case (no duplicate for one firing)", () => {
+  const corpus = buildBacktestCorpus(
+    RULE,
+    [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), fired("a#1", { occurredAt: "2026-07-22T02:30:00.000Z" })],
+    [
+      override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+      override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
+    ],
+  );
+  assert.equal(corpus.length, 2);
+  assert.deepEqual(corpus.map((c) => c.decidedAt), ["2026-07-22T01:00:00.000Z", "2026-07-22T03:00:00.000Z"]);
+});
+
+test("buildBacktestCorpus: fired and override events for a different ruleId are ignored", () => {
+  const corpus = buildBacktestCorpus(
+    RULE,
+    [fired("a#1"), { ruleId: "other_rule", targetKey: "a#1", outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z" }],
+    [override("a#1", "confirmed"), { ruleId: "other_rule", targetKey: "a#1", verdict: "reversed", occurredAt: "2026-07-22T01:00:00.000Z" }],
+  );
+  assert.equal(corpus.length, 1);
+  assert.equal(corpus[0]!.ruleId, RULE);
+  assert.equal(corpus[0]!.label, "confirmed");
+});
+
+test("buildBacktestCorpus: empty input arrays produce an empty corpus", () => {
+  assert.deepEqual(buildBacktestCorpus(RULE, [], []), []);
+});

test/unit/backtest-corpus.test.ts

@@ -0,0 +1,105 @@
+import { describe, expect, it } from "vitest";
+// Direct src-path import (not the `@loopover/engine` package barrel, which resolves to dist and is NOT in
+// vitest's coverage.include): the engine's own node:test suite runs against dist and is invisible to Codecov
+// (only review-enrichment has a c8 dist-remap harvest step; the engine has none), so this vitest test is what
+// gives packages/loopover-engine/src/calibration/backtest-corpus.ts its codecov/patch coverage. The companion
+// packages/loopover-engine/test/backtest-corpus.test.ts is the issue-required node:test that gates the engine
+// workspace's own `npm run test`. Vite resolves the `.js` specifier to the sibling `.ts` on disk.
+import { buildBacktestCorpus } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js";
+import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js";
+import type { HumanOverrideEvent, RuleFiredEvent } from "../../packages/loopover-engine/src/calibration/signal-tracking.js";
+
+const RULE = "missing_linked_issue";
+
+function fired(targetKey: string, overrides: Partial<RuleFiredEvent> = {}): RuleFiredEvent {
+  return { ruleId: RULE, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides };
+}
+
+function override(targetKey: string, verdict: HumanOverrideEvent["verdict"], overrides: Partial<HumanOverrideEvent> = {}): HumanOverrideEvent {
+  return { ruleId: RULE, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides };
+}
+
+describe("buildBacktestCorpus (#8083)", () => {
+  it("excludes a fired event with no matching override (only decided cases count)", () => {
+    const corpus = buildBacktestCorpus(RULE, [fired("a#1"), fired("a#2")], [override("a#1", "confirmed")]);
+    expect(corpus.map((c) => c.targetKey)).toEqual(["a#1"]);
+  });
+
+  it("produces one correctly-labeled case for a single fired+override pair, carrying metadata through", () => {
+    const corpus = buildBacktestCorpus(
+      RULE,
+      [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z", metadata: { pr: 1 } })],
+      [override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" })],
+    );
+    expect(corpus).toEqual([
+      {
+        ruleId: RULE,
+        targetKey: "a#1",
+        outcome: "block",
+        label: "reversed",
+        firedAt: "2026-07-22T00:00:00.000Z",
+        decidedAt: "2026-07-22T02:00:00.000Z",
+        metadata: { pr: 1 },
+      } satisfies BacktestCase,
+    ]);
+  });
+
+  it("omits metadata entirely (never sets it to undefined) when the fired event has none", () => {
+    const corpus = buildBacktestCorpus(RULE, [fired("a#1")], [override("a#1", "confirmed")]);
+    expect("metadata" in corpus[0]!).toBe(false);
+  });
+
+  it("pairs a firing with the nearest override strictly after it when a target was judged multiple times", () => {
+    const corpus = buildBacktestCorpus(
+      RULE,
+      [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" })],
+      [
+        override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
+        override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+        override("a#1", "confirmed", { occurredAt: "2026-07-21T23:00:00.000Z" }),
+      ],
+    );
+    expect(corpus).toHaveLength(1);
+    expect(corpus[0]!.label).toBe("reversed");
+    expect(corpus[0]!.decidedAt).toBe("2026-07-22T01:00:00.000Z");
+  });
+
+  it("falls back to the most recent override when none strictly follows the firing", () => {
+    const corpus = buildBacktestCorpus(
+      RULE,
+      [fired("a#1", { occurredAt: "2026-07-22T05:00:00.000Z" })],
+      [
+        override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+        override("a#1", "confirmed", { occurredAt: "2026-07-22T04:00:00.000Z" }),
+      ],
+    );
+    expect(corpus[0]!.label).toBe("confirmed");
+    expect(corpus[0]!.decidedAt).toBe("2026-07-22T04:00:00.000Z");
+  });
+
+  it("gives each of two firings for the same target its own case (no duplicate for one firing)", () => {
+    const corpus = buildBacktestCorpus(
+      RULE,
+      [fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), fired("a#1", { occurredAt: "2026-07-22T02:30:00.000Z" })],
+      [
+        override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+        override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
+      ],
+    );
+    expect(corpus.map((c) => c.decidedAt)).toEqual(["2026-07-22T01:00:00.000Z", "2026-07-22T03:00:00.000Z"]);
+  });
+
+  it("ignores fired and override events for a different ruleId", () => {
+    const corpus = buildBacktestCorpus(
+      RULE,
+      [fired("a#1"), { ruleId: "other_rule", targetKey: "a#1", outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z" }],
+      [override("a#1", "confirmed"), { ruleId: "other_rule", targetKey: "a#1", verdict: "reversed", occurredAt: "2026-07-22T01:00:00.000Z" }],
+    );
+    expect(corpus).toHaveLength(1);
+    expect(corpus[0]!).toMatchObject({ ruleId: RULE, label: "confirmed" });
+  });
+
+  it("returns an empty corpus for empty input arrays", () => {
+    expect(buildBacktestCorpus(RULE, [], [])).toEqual([]);
+  });
+});