feat(engine): build a labeled backtest corpus from rule-fired and human-override history

packages/loopover-engine/src/calibration/backtest-corpus.ts

@@ -0,0 +1,88 @@
+// Labeled backtest corpus builder (#8083, parent epic #8082) -- turns the calibration module's raw
+// RuleFiredEvent/HumanOverrideEvent history into the per-case labeled records a backtest needs. Where
+// computeRulePrecision (signal-tracking.ts) aggregates the SAME pairing into one precision number, this
+// keeps each paired firing as an individual replayable case: "this rule fired against this target, and a
+// human later said it was right/wrong."
+//
+// SELF-CONTAINED AND PURE, like everything in this module: no IO, no DB, no env -- only the existing
+// event types from signal-tracking.ts. Additive-only; no existing consumer changes behavior.
+
+import type { HumanOverrideEvent, RuleFiredEvent } from "./signal-tracking.js";
+
+/** One labeled, replayable backtest case: a specific rule firing plus the human verdict it eventually got.
+ *  `outcome` is the fired event's outcome; `label` is the override's verdict; `firedAt`/`decidedAt` carry the
+ *  two events' own timestamps. `metadata` is the FIRED event's metadata, omitted entirely (not set to
+ *  `undefined`) when the fired event has none -- the same optional-property discipline RuleFiredEvent uses. */
+export type BacktestCase = {
+  ruleId: string;
+  targetKey: string;
+  outcome: string;
+  label: "reversed" | "confirmed";
+  firedAt: string;
+  decidedAt: string;
+  metadata?: Record<string, unknown>;
+};
+
+/**
+ * Build the labeled corpus for `ruleId` from its fired + override events. A fired event pairs with an
+ * override when both carry the function's `ruleId` AND the same `targetKey`; a fired event with no matching
+ * override is EXCLUDED (not emitted as an unlabeled case) -- mirrors computeRulePrecision's "only the
+ * decided ones count" discipline (see that function's own doc comment in signal-tracking.ts).
+ *
+ * Pairing rule when a target has MULTIPLE overrides for the same rule (re-fired and re-judged more than
+ * once): each fired event pairs with the override whose `occurredAt` is closest in time strictly AFTER that
+ * specific fired event's `occurredAt`; when none strictly follows it, it falls back to the most recent
+ * override by `occurredAt`. Each fired event produces at most one BacktestCase -- never duplicates.
+ */
+export function buildBacktestCorpus(
+  ruleId: string,
+  fired: readonly RuleFiredEvent[],
+  overrides: readonly HumanOverrideEvent[],
+): BacktestCase[] {
+  // Mirrors overrideMatchesRule's one-line filter in signal-tracking.ts (kept private there; this module is
+  // additive-only and must not modify that file to export it).
+  const matchingOverrides = overrides.filter((event) => event.ruleId === ruleId);
+
+  const cases: BacktestCase[] = [];
+  for (const event of fired) {
+    if (event.ruleId !== ruleId) continue;
+    const candidates = matchingOverrides.filter((override) => override.targetKey === event.targetKey);
+    if (candidates.length === 0) continue;
+
+    const firedAtMs = Date.parse(event.occurredAt);
+    let paired: HumanOverrideEvent | undefined;
+    let pairedDeltaMs = Number.POSITIVE_INFINITY;
+    for (const override of candidates) {
+      const deltaMs = Date.parse(override.occurredAt) - firedAtMs;
+      if (deltaMs > 0 && deltaMs < pairedDeltaMs) {
+        paired = override;
+        pairedDeltaMs = deltaMs;
+      }
+    }
+    if (!paired) {
+      // No override strictly follows this firing -- fall back to the most recent override by occurredAt.
+      let latestMs = Number.NEGATIVE_INFINITY;
+      for (const override of candidates) {
+        const occurredMs = Date.parse(override.occurredAt);
+        if (occurredMs > latestMs) {
+          paired = override;
+          latestMs = occurredMs;
+        }
+      }
+    }
+
+    // `paired` is always set here: candidates is non-empty and the fallback scans every candidate with a
+    // strictly-greater-than comparison against -Infinity.
+    const pairedOverride = paired as HumanOverrideEvent;
+    cases.push({
+      ruleId: event.ruleId,
+      targetKey: event.targetKey,
+      outcome: event.outcome,
+      label: pairedOverride.verdict,
+      firedAt: event.occurredAt,
+      decidedAt: pairedOverride.occurredAt,
+      ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
+    });
+  }
+  return cases;
+}

packages/loopover-engine/src/index.ts

@@ -163,6 +163,7 @@ export * from "./governor/kill-switch.js";
 export * from "./governor/action-mode.js";
 export * from "./governor/chokepoint.js";
 export * from "./calibration/signal-tracking.js";
+export * from "./calibration/backtest-corpus.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/backtest-corpus.test.ts

@@ -0,0 +1,135 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "../dist/index.js";
+
+// #8083: buildBacktestCorpus pairs each rule firing with its human verdict into a labeled, replayable
+// BacktestCase. Mirrors signal-tracking.test.ts's fixture style; the pairing rule under test is the
+// nearest-strictly-following override, falling back to the most recent when none follows.
+
+function fired(ruleId: string, targetKey: string, overrides: Partial<RuleFiredEvent> = {}): RuleFiredEvent {
+  return { ruleId, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides };
+}
+
+function override(
+  ruleId: string,
+  targetKey: string,
+  verdict: HumanOverrideEvent["verdict"],
+  overrides: Partial<HumanOverrideEvent> = {},
+): HumanOverrideEvent {
+  return { ruleId, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides };
+}
+
+test("barrel: the public entrypoint re-exports the backtest-corpus builder (#8083)", () => {
+  assert.equal(typeof buildBacktestCorpus, "function");
+});
+
+test("a single fired+override pair produces one correctly-labeled case", () => {
+  const corpus = buildBacktestCorpus(
+    "missing_linked_issue",
+    [fired("missing_linked_issue", "a/r#1", { metadata: { headSha: "abc" } })],
+    [override("missing_linked_issue", "a/r#1", "reversed")],
+  );
+  assert.deepEqual(corpus, [
+    {
+      ruleId: "missing_linked_issue",
+      targetKey: "a/r#1",
+      outcome: "block",
+      label: "reversed",
+      firedAt: "2026-07-22T00:00:00.000Z",
+      decidedAt: "2026-07-22T01:00:00.000Z",
+      metadata: { headSha: "abc" },
+    } satisfies BacktestCase,
+  ]);
+});
+
+test("a fired event with no matching override is excluded, not emitted unlabeled", () => {
+  const corpus = buildBacktestCorpus(
+    "missing_linked_issue",
+    [fired("missing_linked_issue", "a/r#1"), fired("missing_linked_issue", "a/r#2")],
+    [override("missing_linked_issue", "a/r#2", "confirmed")],
+  );
+  assert.deepEqual(
+    corpus.map((backtestCase) => backtestCase.targetKey),
+    ["a/r#2"],
+  );
+});
+
+test("metadata is omitted entirely (not set to undefined) when the fired event has none", () => {
+  const [backtestCase] = buildBacktestCorpus(
+    "missing_linked_issue",
+    [fired("missing_linked_issue", "a/r#1")],
+    [override("missing_linked_issue", "a/r#1", "confirmed")],
+  );
+  assert.equal(Object.hasOwn(backtestCase!, "metadata"), false);
+});
+
+test("multiple overrides for one target pair each firing with the nearest strictly-following override", () => {
+  const corpus = buildBacktestCorpus(
+    "rule",
+    [
+      fired("rule", "a/r#1", { occurredAt: "2026-07-22T00:00:00.000Z" }),
+      fired("rule", "a/r#1", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+    ],
+    [
+      override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+      override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
+    ],
+  );
+  assert.deepEqual(
+    corpus.map((backtestCase) => [backtestCase.label, backtestCase.decidedAt]),
+    [
+      ["reversed", "2026-07-22T01:00:00.000Z"],
+      ["confirmed", "2026-07-22T03:00:00.000Z"],
+    ],
+  );
+});
+
+test("a firing with no strictly-following override falls back to the most recent override", () => {
+  const corpus = buildBacktestCorpus(
+    "rule",
+    [fired("rule", "a/r#1", { occurredAt: "2026-07-22T05:00:00.000Z" })],
+    [
+      override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+      override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
+    ],
+  );
+  assert.deepEqual(
+    corpus.map((backtestCase) => [backtestCase.label, backtestCase.decidedAt]),
+    [["confirmed", "2026-07-22T03:00:00.000Z"]],
+  );
+});
+
+test("an override at exactly the firing's own instant does not count as strictly following", () => {
+  const corpus = buildBacktestCorpus(
+    "rule",
+    [fired("rule", "a/r#1", { occurredAt: "2026-07-22T02:00:00.000Z" })],
+    [
+      override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+      override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+    ],
+  );
+  // Neither override strictly follows, so the most recent one (02:00, "confirmed") wins the fallback.
+  assert.deepEqual(
+    corpus.map((backtestCase) => [backtestCase.label, backtestCase.decidedAt]),
+    [["confirmed", "2026-07-22T02:00:00.000Z"]],
+  );
+});
+
+test("events for a different ruleId are ignored on both sides", () => {
+  const corpus = buildBacktestCorpus(
+    "rule",
+    [fired("other_rule", "a/r#1"), fired("rule", "a/r#2")],
+    [override("rule", "a/r#1", "reversed"), override("other_rule", "a/r#2", "reversed"), override("rule", "a/r#2", "confirmed")],
+  );
+  assert.deepEqual(
+    corpus.map((backtestCase) => [backtestCase.targetKey, backtestCase.label]),
+    [["a/r#2", "confirmed"]],
+  );
+});
+
+test("empty input arrays produce an empty corpus", () => {
+  assert.deepEqual(buildBacktestCorpus("rule", [], []), []);
+  assert.deepEqual(buildBacktestCorpus("rule", [fired("rule", "a/r#1")], []), []);
+  assert.deepEqual(buildBacktestCorpus("rule", [], [override("rule", "a/r#1", "reversed")]), []);
+});

test/unit/backtest-corpus.test.ts

@@ -0,0 +1,118 @@
+import { describe, expect, it } from "vitest";
+import {
+  buildBacktestCorpus,
+  type BacktestCase,
+} from "../../packages/loopover-engine/src/calibration/backtest-corpus.js";
+import type { HumanOverrideEvent, RuleFiredEvent } from "../../packages/loopover-engine/src/calibration/signal-tracking.js";
+
+// #8083: root-side coverage twin of packages/loopover-engine/test/backtest-corpus.test.ts. The engine
+// package's own node:test suite runs against dist/ (not instrumented by the root vitest coverage that
+// Codecov gates on), so this file exercises the SAME contract directly against the engine src — the same
+// direct-src import pattern test/contract/*-parity.test.ts and test/integration/miner-*.test.ts already use.
+
+function fired(ruleId: string, targetKey: string, overrides: Partial<RuleFiredEvent> = {}): RuleFiredEvent {
+  return { ruleId, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides };
+}
+
+function override(
+  ruleId: string,
+  targetKey: string,
+  verdict: HumanOverrideEvent["verdict"],
+  overrides: Partial<HumanOverrideEvent> = {},
+): HumanOverrideEvent {
+  return { ruleId, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides };
+}
+
+describe("buildBacktestCorpus (#8083)", () => {
+  it("pairs a single fired+override into one labeled case, carrying outcome, timestamps, and metadata", () => {
+    const corpus = buildBacktestCorpus(
+      "missing_linked_issue",
+      [fired("missing_linked_issue", "a/r#1", { outcome: "exclude", metadata: { headSha: "abc" } })],
+      [override("missing_linked_issue", "a/r#1", "reversed")],
+    );
+    expect(corpus).toEqual([
+      {
+        ruleId: "missing_linked_issue",
+        targetKey: "a/r#1",
+        outcome: "exclude",
+        label: "reversed",
+        firedAt: "2026-07-22T00:00:00.000Z",
+        decidedAt: "2026-07-22T01:00:00.000Z",
+        metadata: { headSha: "abc" },
+      } satisfies BacktestCase,
+    ]);
+  });
+
+  it("excludes fired events with no matching override — only the decided ones count", () => {
+    const corpus = buildBacktestCorpus(
+      "rule",
+      [fired("rule", "a/r#1"), fired("rule", "a/r#2")],
+      [override("rule", "a/r#2", "confirmed")],
+    );
+    expect(corpus.map((backtestCase) => backtestCase.targetKey)).toEqual(["a/r#2"]);
+  });
+
+  it("omits the metadata key entirely when the fired event has none", () => {
+    const [backtestCase] = buildBacktestCorpus(
+      "rule",
+      [fired("rule", "a/r#1")],
+      [override("rule", "a/r#1", "confirmed")],
+    );
+    expect(Object.hasOwn(backtestCase!, "metadata")).toBe(false);
+  });
+
+  it("pairs each firing with the nearest strictly-following override when a target was judged repeatedly", () => {
+    const corpus = buildBacktestCorpus(
+      "rule",
+      [
+        fired("rule", "a/r#1", { occurredAt: "2026-07-22T00:00:00.000Z" }),
+        fired("rule", "a/r#1", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+      ],
+      [
+        override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+        override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
+      ],
+    );
+    expect(corpus.map((backtestCase) => [backtestCase.label, backtestCase.decidedAt])).toEqual([
+      ["reversed", "2026-07-22T01:00:00.000Z"],
+      ["confirmed", "2026-07-22T03:00:00.000Z"],
+    ]);
+  });
+
+  it("falls back to the most recent override when none strictly follows the firing (equal instants included)", () => {
+    const late = buildBacktestCorpus(
+      "rule",
+      [fired("rule", "a/r#1", { occurredAt: "2026-07-22T05:00:00.000Z" })],
+      [
+        override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+        override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
+      ],
+    );
+    expect(late.map((backtestCase) => backtestCase.decidedAt)).toEqual(["2026-07-22T03:00:00.000Z"]);
+
+    const equalInstant = buildBacktestCorpus(
+      "rule",
+      [fired("rule", "a/r#1", { occurredAt: "2026-07-22T02:00:00.000Z" })],
+      [
+        override("rule", "a/r#1", "confirmed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+        override("rule", "a/r#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+      ],
+    );
+    expect(equalInstant.map((backtestCase) => backtestCase.label)).toEqual(["confirmed"]);
+  });
+
+  it("ignores events for a different ruleId on both the fired and override sides", () => {
+    const corpus = buildBacktestCorpus(
+      "rule",
+      [fired("other_rule", "a/r#1"), fired("rule", "a/r#2")],
+      [override("rule", "a/r#1", "reversed"), override("other_rule", "a/r#2", "reversed"), override("rule", "a/r#2", "confirmed")],
+    );
+    expect(corpus.map((backtestCase) => [backtestCase.targetKey, backtestCase.label])).toEqual([["a/r#2", "confirmed"]]);
+  });
+
+  it("produces an empty corpus for empty inputs in every combination", () => {
+    expect(buildBacktestCorpus("rule", [], [])).toEqual([]);
+    expect(buildBacktestCorpus("rule", [fired("rule", "a/r#1")], [])).toEqual([]);
+    expect(buildBacktestCorpus("rule", [], [override("rule", "a/r#1", "reversed")])).toEqual([]);
+  });
+});