From 3142b4ef4cceb13ac70c2f969b8f6fe068b3d996 Mon Sep 17 00:00:00 2001 From: RealDiligent Date: Thu, 23 Jul 2026 22:33:58 +0800 Subject: [PATCH] feat(calibration): pure counterfactual fixture assembler per the replay contract (#8220) --- .../calibration/counterfactual-fixtures.ts | 84 ++++++++++++++++ packages/loopover-engine/src/index.ts | 1 + .../test/counterfactual-fixtures.test.ts | 49 ++++++++++ .../counterfactual-fixtures-engine.test.ts | 96 +++++++++++++++++++ 4 files changed, 230 insertions(+) create mode 100644 packages/loopover-engine/src/calibration/counterfactual-fixtures.ts create mode 100644 packages/loopover-engine/test/counterfactual-fixtures.test.ts create mode 100644 test/unit/counterfactual-fixtures-engine.test.ts diff --git a/packages/loopover-engine/src/calibration/counterfactual-fixtures.ts b/packages/loopover-engine/src/calibration/counterfactual-fixtures.ts new file mode 100644 index 0000000000..fc26b6e8ab --- /dev/null +++ b/packages/loopover-engine/src/calibration/counterfactual-fixtures.ts @@ -0,0 +1,84 @@ +// Counterfactual fixture assembler (#8220, sub-epic phase 2): from a labeled corpus, select and shape +// exactly the cases that are replayable under the #8219 contract — pure selection, no AI calls, no IO. +// Implements the contract's fixture/sampling shapes VERBATIM (counterfactual-contract.ts); every exclusion +// is accounted for in the return shape (the #8139 skipped-case discipline), never silently dropped. +// +// PROVENANCE: the #8207/#8170 backfill patches a re-fetched row's metadata with a `rawContextProvenance` +// tag (RAW_CONTEXT_REFETCH_PROVENANCE in scripts/backfill-calibration-corpus-phase2-core.ts); the live +// #8129/#8130 capture writers never set that key. Presence of the key is therefore the era discriminator — +// any tagged row is backfilled context, an untagged replayable row is live-captured. +// +// SAMPLING: when the eligible set exceeds the contract's budget, membership is decided by the same +// content-hash discipline as splitBacktestCorpus — sha256(`${seed}:${targetKey}`), first 8 hex chars as the +// rank, lowest ranks win — never "the first N", which would bias toward old cases. Selection keeps the +// corpus's own case order (no shuffle); a rank tie (two firings of the SAME target share a hash) breaks +// toward the earlier case, deterministically. + +import { createHash } from "node:crypto"; +import type { BacktestCase } from "./backtest-corpus.js"; +import { + isReplayableCase, + type CounterfactualFixture, + type CounterfactualSamplingContract, + type CounterfactualSkipReason, +} from "./counterfactual-contract.js"; + +export type CounterfactualFixtureAssembly = { + /** Replayable fixtures in corpus order, at most `contract.maxFixtures` of them. */ + fixtures: CounterfactualFixture[]; + /** Why every non-fixture case was excluded — `fixtures.length` plus these counts always sums to the + * input corpus size (pinned by an invariant test). */ + skipped: Record; +}; + +function sampleRank(seed: string, targetKey: string): number { + return parseInt(createHash("sha256").update(`${seed}:${targetKey}`).digest("hex").slice(0, 8), 16); +} + +function toFixture(backtestCase: BacktestCase): CounterfactualFixture { + // isReplayableCase already guaranteed a non-empty string diff for every case reaching here. + const metadata = backtestCase.metadata!; + return { + fixtureId: backtestCase.targetKey, + label: backtestCase.label, + boundedInputs: { diff: metadata.diff as string }, + provenance: "rawContextProvenance" in metadata ? "raw_context_refetch" : "live_capture", + }; +} + +/** + * Assemble the replayable fixture set for one campaign per the #8219 contract: filter to cases carrying + * bounded raw context (via the contract's own {@link isReplayableCase}), apply the seeded deterministic + * sample when the eligible set exceeds `contract.maxFixtures`, and emit fixtures with era provenance — + * with full skip accounting for everything excluded. Deterministic: same corpus + contract ⇒ same fixture + * set, byte for byte. + */ +export function assembleCounterfactualFixtures( + cases: readonly BacktestCase[], + contract: CounterfactualSamplingContract, +): CounterfactualFixtureAssembly { + const skipped: Record = { no_raw_context: 0, sampled_out: 0 }; + const eligible: BacktestCase[] = []; + for (const backtestCase of cases) { + if (!isReplayableCase(backtestCase)) { + skipped.no_raw_context += 1; + continue; + } + eligible.push(backtestCase); + } + + let selected = eligible; + if (eligible.length > contract.maxFixtures) { + const kept = new Set( + eligible + .map((backtestCase, index) => ({ index, rank: sampleRank(contract.seed, backtestCase.targetKey) })) + .sort((a, b) => a.rank - b.rank || a.index - b.index) + .slice(0, contract.maxFixtures) + .map((entry) => entry.index), + ); + selected = eligible.filter((_, index) => kept.has(index)); + skipped.sampled_out = eligible.length - selected.length; + } + + return { fixtures: selected.map(toFixture), skipped }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index beef2399fc..5db7f90887 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -167,6 +167,7 @@ export * from "./calibration/backtest-corpus.js"; export * from "./calibration/repo-corpus-slice.js"; export * from "./calibration/ams-prediction-corpus.js"; export * from "./calibration/counterfactual-contract.js"; +export * from "./calibration/counterfactual-fixtures.js"; export * from "./calibration/backtest-score.js"; export * from "./calibration/backtest-compare.js"; export * from "./calibration/backtest-report.js"; diff --git a/packages/loopover-engine/test/counterfactual-fixtures.test.ts b/packages/loopover-engine/test/counterfactual-fixtures.test.ts new file mode 100644 index 0000000000..7d4bc916b6 --- /dev/null +++ b/packages/loopover-engine/test/counterfactual-fixtures.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + assembleCounterfactualFixtures, + COUNTERFACTUAL_SAMPLE_SEED_PREFIX, + type BacktestCase, +} from "../dist/index.js"; + +const SEED = `${COUNTERFACTUAL_SAMPLE_SEED_PREFIX}:workspace-suite`; + +function replayable(targetKey: string, label: BacktestCase["label"] = "confirmed", extraMetadata: Record = {}): BacktestCase { + return { + ruleId: "ai_consensus_defect", + targetKey, + outcome: "close", + label, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + metadata: { diff: `@@ diff for ${targetKey}`, ...extraMetadata }, + }; +} + +test("barrel: the public entrypoint re-exports the fixture assembler (#8220)", () => { + assert.equal(typeof assembleCounterfactualFixtures, "function"); +}); + +test("assembler round-trip: eligibility, era provenance, and skip accounting per the #8219 contract", () => { + const { fixtures, skipped } = assembleCounterfactualFixtures( + [ + replayable("acme/widgets#1", "reversed"), + { ...replayable("acme/widgets#2"), metadata: { confidence: 0.5 } }, + replayable("acme/widgets#3", "confirmed", { rawContextProvenance: "github_raw_context_refetch" }), + ], + { seed: SEED, maxFixtures: 10 }, + ); + assert.equal(fixtures.length, 2); + assert.equal(fixtures[0]!.provenance, "live_capture"); + assert.equal(fixtures[1]!.provenance, "raw_context_refetch"); + assert.deepEqual(skipped, { no_raw_context: 1, sampled_out: 0 }); +}); + +test("seeded sampling is deterministic and accounts every sampled-out case", () => { + const cases = Array.from({ length: 25 }, (_, i) => replayable(`acme/widgets#${i + 1}`)); + const first = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 9 }); + assert.equal(first.fixtures.length, 9); + assert.equal(first.skipped.sampled_out, 16); + assert.deepEqual(assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 9 }), first); +}); diff --git a/test/unit/counterfactual-fixtures-engine.test.ts b/test/unit/counterfactual-fixtures-engine.test.ts new file mode 100644 index 0000000000..7cdba1e5e8 --- /dev/null +++ b/test/unit/counterfactual-fixtures-engine.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; + +// Import the engine SOURCE directly (not the built dist) -- coverage.include lists +// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in +// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace +// suite). Same pattern as backtest-corpus-engine.test.ts / repo-corpus-engine.test.ts. +import { assembleCounterfactualFixtures } from "../../packages/loopover-engine/src/calibration/counterfactual-fixtures"; +import { COUNTERFACTUAL_SAMPLE_SEED_PREFIX } from "../../packages/loopover-engine/src/calibration/counterfactual-contract"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus"; + +const SEED = `${COUNTERFACTUAL_SAMPLE_SEED_PREFIX}:test-campaign`; + +function replayable(targetKey: string, label: BacktestCase["label"] = "confirmed", extraMetadata: Record = {}): BacktestCase { + return { + ruleId: "ai_consensus_defect", + targetKey, + outcome: "close", + label, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + metadata: { diff: `@@ diff for ${targetKey}`, ...extraMetadata }, + }; +} + +describe("assembleCounterfactualFixtures (#8220)", () => { + it("shapes replayable cases into fixtures with era provenance, skipping context-less cases with accounting", () => { + const cases: BacktestCase[] = [ + replayable("acme/widgets#1", "reversed"), + { ...replayable("acme/widgets#2"), metadata: { confidence: 0.9 } }, // no diff — not replayable + { ...replayable("acme/widgets#3"), metadata: { diff: "" } }, // empty diff — not replayable + replayable("acme/widgets#4", "confirmed", { rawContextProvenance: "github_raw_context_refetch" }), + ]; + const { fixtures, skipped } = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 10 }); + expect(fixtures).toEqual([ + { + fixtureId: "acme/widgets#1", + label: "reversed", + boundedInputs: { diff: "@@ diff for acme/widgets#1" }, + provenance: "live_capture", + }, + { + fixtureId: "acme/widgets#4", + label: "confirmed", + boundedInputs: { diff: "@@ diff for acme/widgets#4" }, + provenance: "raw_context_refetch", + }, + ]); + expect(skipped).toEqual({ no_raw_context: 2, sampled_out: 0 }); + // Sum invariant: every input case is a fixture or an accounted skip. + expect(fixtures.length + skipped.no_raw_context + skipped.sampled_out).toBe(cases.length); + }); + + it("applies the seeded sample only when the eligible set exceeds the budget, preserving corpus order", () => { + const cases = Array.from({ length: 20 }, (_, i) => replayable(`acme/widgets#${i + 1}`)); + const under = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 20 }); + expect(under.fixtures).toHaveLength(20); + expect(under.skipped.sampled_out).toBe(0); + + const sampled = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 7 }); + expect(sampled.fixtures).toHaveLength(7); + expect(sampled.skipped.sampled_out).toBe(13); + // Corpus order is preserved within the sample — fixture ids ascend by original position. + const positions = sampled.fixtures.map((fixture) => cases.findIndex((c) => c.targetKey === fixture.fixtureId)); + expect(positions).toEqual([...positions].sort((a, b) => a - b)); + }); + + it("is deterministic per seed, differs across seeds, and never selects 'the first N'", () => { + const cases = Array.from({ length: 30 }, (_, i) => replayable(`acme/widgets#${i + 1}`)); + const first = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 10 }); + expect(assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 10 })).toEqual(first); + + const otherSeed = assembleCounterfactualFixtures(cases, { seed: `${SEED}-b`, maxFixtures: 10 }); + expect(otherSeed.fixtures.map((f) => f.fixtureId)).not.toEqual(first.fixtures.map((f) => f.fixtureId)); + // Hash-ranked membership, not positional truncation. + expect(first.fixtures.map((f) => f.fixtureId)).not.toEqual(cases.slice(0, 10).map((c) => c.targetKey)); + }); + + it("breaks a same-target rank tie toward the earlier case, deterministically", () => { + // Two firings of the SAME target share a sample hash — a two-element sort MUST compare exactly that + // tied pair, forcing the position tie-break: the earlier firing wins the single slot. + const cases = [replayable("acme/widgets#7", "confirmed"), replayable("acme/widgets#7", "reversed")]; + const { fixtures, skipped } = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 1 }); + expect(fixtures).toHaveLength(1); + expect(fixtures[0]).toMatchObject({ fixtureId: "acme/widgets#7", label: "confirmed" }); + expect(skipped.sampled_out).toBe(1); + // Reproducible byte-for-byte. + expect(assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 1 })).toEqual({ fixtures, skipped }); + }); + + it("returns an empty assembly for an empty corpus", () => { + expect(assembleCounterfactualFixtures([], { seed: SEED, maxFixtures: 5 })).toEqual({ + fixtures: [], + skipped: { no_raw_context: 0, sampled_out: 0 }, + }); + }); +});