diff --git a/packages/loopover-engine/src/calibration/backtest-split.ts b/packages/loopover-engine/src/calibration/backtest-split.ts new file mode 100644 index 0000000000..0cfc77c91c --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-split.ts @@ -0,0 +1,40 @@ +// Deterministic held-out/visible corpus split (#8087) -- the dual-target evaluation method: iterate a +// candidate rule against the visible slice, score it against BOTH slices, so a fix can't be hand-tuned to +// just the specific incidents already known about. Reuses the same content-hash approach as +// stableProposalId in ../miner/deny-hook-synthesis.ts (sha256 over a composite key): a case's assignment +// depends only on (seed, ruleId, targetKey) -- never on its position or on cases.length -- so a corpus +// that grows over time never reshuffles which already-processed cases were previously held out. +// +// Same purity contract as the rest of this module family: no IO, no Math.random(), no wall-clock reads. + +import { createHash } from "node:crypto"; +import type { BacktestCase } from "./backtest-corpus.js"; + +/** + * Partition `cases` into a visible slice and a held-out slice of roughly `heldOutFraction` of the corpus. + * Deterministic: sha256(`${seed}:${ruleId}:${targetKey}`), first 8 hex chars as a base-16 integer over + * 0xffffffff, held out when strictly below `heldOutFraction` -- identical inputs always produce + * byte-identical output. Each case keeps its original input-order position within its assigned bucket + * (no sorting, no shuffling). Throws when `heldOutFraction` is outside the inclusive [0, 1] range. + */ +export function splitBacktestCorpus( + cases: readonly BacktestCase[], + heldOutFraction: number, + seed: string, +): { visible: BacktestCase[]; heldOut: BacktestCase[] } { + // Negated compound form so a NaN fraction also fails closed instead of silently splitting nothing out. + if (!(heldOutFraction >= 0 && heldOutFraction <= 1)) { + throw new Error(`invalid_held_out_fraction: ${heldOutFraction}`); + } + const visible: BacktestCase[] = []; + const heldOut: BacktestCase[] = []; + for (const backtestCase of cases) { + const digest = createHash("sha256") + .update(`${seed}:${backtestCase.ruleId}:${backtestCase.targetKey}`) + .digest("hex"); + const value = parseInt(digest.slice(0, 8), 16) / 0xffffffff; + if (value < heldOutFraction) heldOut.push(backtestCase); + else visible.push(backtestCase); + } + return { visible, heldOut }; +} diff --git a/packages/loopover-engine/test/backtest-split.test.ts b/packages/loopover-engine/test/backtest-split.test.ts new file mode 100644 index 0000000000..eb56313df3 --- /dev/null +++ b/packages/loopover-engine/test/backtest-split.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import type { BacktestCase } from "../dist/index.js"; +import { splitBacktestCorpus } from "../dist/calibration/backtest-split.js"; + +function corpusCase(targetKey: string, overrides: Partial = {}): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey, + outcome: "block", + label: "confirmed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + ...overrides, + }; +} + +/** A dozen distinct targets -- enough that a 0.5 split reliably lands cases in BOTH buckets and that two + * different seeds reliably disagree on at least one case, without depending on any specific hash value. */ +const corpus = Array.from({ length: 12 }, (_, index) => corpusCase(`acme/widgets#${index + 1}`)); + +test("splitBacktestCorpus: heldOutFraction 0 keeps every case visible", () => { + const { visible, heldOut } = splitBacktestCorpus(corpus, 0, "seed-a"); + assert.deepEqual(visible, corpus); + assert.deepEqual(heldOut, []); +}); + +test("splitBacktestCorpus: heldOutFraction 1 holds every case out", () => { + const { visible, heldOut } = splitBacktestCorpus(corpus, 1, "seed-a"); + assert.deepEqual(heldOut, corpus); + assert.deepEqual(visible, []); +}); + +test("splitBacktestCorpus: identical inputs produce byte-identical output, including per-bucket order", () => { + const first = splitBacktestCorpus(corpus, 0.5, "seed-a"); + const second = splitBacktestCorpus(corpus, 0.5, "seed-a"); + assert.deepEqual(second, first); +}); + +test("splitBacktestCorpus: preserves each case's original input order within its bucket, with both buckets populated", () => { + const { visible, heldOut } = splitBacktestCorpus(corpus, 0.5, "seed-a"); + assert.ok(visible.length > 0 && heldOut.length > 0, "0.5 over 12 distinct targets must populate both buckets"); + const inputIndex = (backtestCase: BacktestCase) => corpus.indexOf(backtestCase); + for (const bucket of [visible, heldOut]) { + const order = bucket.map(inputIndex); + assert.deepEqual(order, [...order].sort((a, b) => a - b)); + } +}); + +test("splitBacktestCorpus: a different seed produces a different split for at least one case", () => { + const first = splitBacktestCorpus(corpus, 0.5, "seed-a"); + const second = splitBacktestCorpus(corpus, 0.5, "seed-b"); + assert.notDeepEqual( + { visible: first.visible.map((c) => c.targetKey), heldOut: first.heldOut.map((c) => c.targetKey) }, + { visible: second.visible.map((c) => c.targetKey), heldOut: second.heldOut.map((c) => c.targetKey) }, + ); +}); + +test("splitBacktestCorpus: throws on an out-of-range heldOutFraction in both directions, naming the value", () => { + assert.throws(() => splitBacktestCorpus(corpus, -0.1, "seed-a"), /invalid_held_out_fraction: -0\.1/); + assert.throws(() => splitBacktestCorpus(corpus, 1.5, "seed-a"), /invalid_held_out_fraction: 1\.5/); +}); diff --git a/test/unit/backtest-split-engine.test.ts b/test/unit/backtest-split-engine.test.ts new file mode 100644 index 0000000000..d03153820b --- /dev/null +++ b/test/unit/backtest-split-engine.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; + +// Import the engine SOURCE directly (not the built dist) -- coverage.include lists +// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in +// (the dist-importing twin in packages/loopover-engine/test/ covers the built artifact for the workspace +// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts. +import { splitBacktestCorpus } from "../../packages/loopover-engine/src/calibration/backtest-split"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus"; + +function corpusCase(targetKey: string): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey, + outcome: "block", + label: "confirmed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + }; +} + +// A dozen distinct targets -- enough that a 0.5 split reliably populates BOTH buckets and two different +// seeds reliably disagree on at least one case, without depending on any specific hash value. +const corpus = Array.from({ length: 12 }, (_, index) => corpusCase(`acme/widgets#${index + 1}`)); + +describe("splitBacktestCorpus (#8087)", () => { + it("keeps every case visible at fraction 0 and holds every case out at fraction 1", () => { + expect(splitBacktestCorpus(corpus, 0, "seed-a")).toEqual({ visible: corpus, heldOut: [] }); + expect(splitBacktestCorpus(corpus, 1, "seed-a")).toEqual({ visible: [], heldOut: corpus }); + }); + + it("is deterministic: identical inputs produce byte-identical output, including per-bucket order", () => { + const first = splitBacktestCorpus(corpus, 0.5, "seed-a"); + expect(splitBacktestCorpus(corpus, 0.5, "seed-a")).toEqual(first); + }); + + it("preserves original input order within each bucket, with both buckets populated at 0.5", () => { + const { visible, heldOut } = splitBacktestCorpus(corpus, 0.5, "seed-a"); + expect(visible.length).toBeGreaterThan(0); + expect(heldOut.length).toBeGreaterThan(0); + for (const bucket of [visible, heldOut]) { + const order = bucket.map((backtestCase) => corpus.indexOf(backtestCase)); + expect(order).toEqual([...order].sort((a, b) => a - b)); + } + }); + + it("produces a different split for at least one case when only the seed changes", () => { + const withSeedA = splitBacktestCorpus(corpus, 0.5, "seed-a"); + const withSeedB = splitBacktestCorpus(corpus, 0.5, "seed-b"); + expect(withSeedB.heldOut.map((c) => c.targetKey)).not.toEqual(withSeedA.heldOut.map((c) => c.targetKey)); + }); + + it("throws on an out-of-range heldOutFraction in both directions, naming the invalid value", () => { + expect(() => splitBacktestCorpus(corpus, -0.1, "seed-a")).toThrow("invalid_held_out_fraction: -0.1"); + expect(() => splitBacktestCorpus(corpus, 1.5, "seed-a")).toThrow("invalid_held_out_fraction: 1.5"); + }); +});