From 994e9dcf57f8f40e54f91399f3927affb630c299 Mon Sep 17 00:00:00 2001 From: Andriy Polanski Date: Wed, 22 Jul 2026 22:23:21 +0000 Subject: [PATCH] feat(scripts): export rule-precision backtest corpus as checksummed JSON (#8084) Add a pure buildBacktestCorpusManifest/checksumCases core (mirroring export-d1-core) plus a thin wrangler D1 CLI that snapshots one rule's fired/override audit_events into a portable labeled corpus file. Co-authored-by: Cursor --- scripts/backtest-corpus-export-core.ts | 44 +++++ scripts/backtest-corpus-export.ts | 160 ++++++++++++++++++ test/unit/backtest-corpus-export-core.test.ts | 74 ++++++++ 3 files changed, 278 insertions(+) create mode 100644 scripts/backtest-corpus-export-core.ts create mode 100644 scripts/backtest-corpus-export.ts create mode 100644 test/unit/backtest-corpus-export-core.test.ts diff --git a/scripts/backtest-corpus-export-core.ts b/scripts/backtest-corpus-export-core.ts new file mode 100644 index 0000000000..a68e97d6f9 --- /dev/null +++ b/scripts/backtest-corpus-export-core.ts @@ -0,0 +1,44 @@ +// Pure core for the rule-precision backtest corpus export (#8084, part of epic #8082). Transforms an already- +// built BacktestCase[] (from buildBacktestCorpus) into a versioned, checksummed manifest a scorer can reload +// without re-querying D1. No IO here — the CLI (backtest-corpus-export.ts) does the wrangler/D1 reads and the +// file write — so this stays unit-testable. Mirrors scripts/export-d1-core.ts's pure-core / thin-IO split. +import { createHash } from "node:crypto"; +import type { BacktestCase } from "@loopover/engine"; + +export type BacktestCorpusManifest = { + ruleId: string; + caseCount: number; + checksum: string; + cases: BacktestCase[]; +}; + +/** Canonicalize one case (sort keys) so property-order differences don't change the checksum — same technique + * as export-d1-core.ts's canonicalizeRow. */ +function canonicalizeCase(backtestCase: BacktestCase): Record { + return Object.fromEntries(Object.entries(backtestCase).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); +} + +/** Deterministic SHA-256 over the canonicalized cases — mirrors export-d1-core.ts's checksumRows exactly + * (canonicalize each entry, JSON-stringify the array, hash). */ +export function checksumCases(cases: readonly BacktestCase[]): string { + return createHash("sha256").update(JSON.stringify(cases.map(canonicalizeCase))).digest("hex"); +} + +/** + * Build the export manifest for one rule's labeled corpus. Spreads an optional `meta` bag into the result + * (the CLI attaches `generatedAt`); this core never reads the clock. Mirrors export-d1-core.ts's + * {@link buildExportManifest} signature shape. + */ +export function buildBacktestCorpusManifest( + ruleId: string, + cases: readonly BacktestCase[], + meta: Record = {}, +): BacktestCorpusManifest & Record { + return { + ...meta, + ruleId, + caseCount: cases.length, + checksum: checksumCases(cases), + cases: [...cases], + }; +} diff --git a/scripts/backtest-corpus-export.ts b/scripts/backtest-corpus-export.ts new file mode 100644 index 0000000000..3309417b09 --- /dev/null +++ b/scripts/backtest-corpus-export.ts @@ -0,0 +1,160 @@ +#!/usr/bin/env node +// Read-only ORB D1 → rule-precision backtest corpus export (#8084, epic #8082). Queries audit_events for one +// rule's `signal.rule_fired:` / `signal.human_override:` rows via `wrangler d1 execute --json` +// (no writes), reconstructs RuleFiredEvent/HumanOverrideEvent the same way signal-tracking-wire.ts does, runs +// buildBacktestCorpus, and writes a checksummed JSON snapshot. The pure transform lives in +// backtest-corpus-export-core.ts (unit-tested); this file is the thin IO wrapper — mirrors export-d1-data.ts. +// +// tsx scripts/backtest-corpus-export.ts --rule-id --output [--remote] [--since-date ] [--db loopover] +// +// --remote reads the deployed D1 (default is the local miniflare DB). --since-date does an INCREMENTAL export +// (rows whose created_at is >= the date); omit it for a full history. NEVER pass a write command. ORB only — +// AMS's event-ledger export is out of scope. +import { writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "@loopover/engine"; +import { buildBacktestCorpusManifest } from "./backtest-corpus-export-core.js"; + +type D1Row = Record; + +type Args = { + ruleId: string | undefined; + output: string | undefined; + remote: boolean; + sinceDate: string | undefined; + db: string; +}; + +const RULE_FIRED_EVENT_TYPE_PREFIX = "signal.rule_fired:"; +const HUMAN_OVERRIDE_EVENT_TYPE_PREFIX = "signal.human_override:"; + +function parseArgs(argv: string[]): Args { + const args: Args = { ruleId: undefined, output: undefined, remote: false, sinceDate: undefined, db: "loopover" }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + if (flag === "--remote") args.remote = true; + else if (flag === "--rule-id") args.ruleId = argv[++i]; + else if (flag === "--output") args.output = argv[++i]; + else if (flag === "--since-date") args.sinceDate = argv[++i]; + else if (flag === "--db") args.db = argv[++i]!; + } + return args; +} + +// Run a read-only SQL statement via wrangler and return the result rows. Throws on any wrangler failure so a +// partial/garbled export can never be mistaken for a complete one. Mirrors export-d1-data.ts's d1Query. +function d1Query(db: string, remote: boolean, sql: string): D1Row[] { + const result = spawnSync("npx", ["wrangler", "d1", "execute", db, remote ? "--remote" : "--local", "--json", "--command", sql], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`wrangler d1 execute failed (${result.status}): ${(result.stderr || result.stdout || "").slice(0, 500)}`); + } + const parsed = JSON.parse(result.stdout); + // wrangler returns [{ results: [...], success, meta }] (one entry per statement). + const first = Array.isArray(parsed) ? parsed[0] : parsed; + return first?.results ?? []; +} + +function sqlStringLiteral(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} + +function parseMetadataJson(raw: unknown): Record { + if (typeof raw !== "string") return {}; + try { + const parsed: unknown = JSON.parse(raw); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record; + } catch { + /* corrupt row -- fail open to {} (mirrors listAuditEventsByType) */ + } + return {}; +} + +// Mirrors src/review/signal-tracking-wire.ts's toRuleFiredEvent — keep in sync with that adapter; do not import +// it (private to the live ORB adapter; this CLI is a read-only export path). +function toRuleFiredEvent(ruleId: string, row: { targetKey: string | null; metadata: Record; createdAt: string }): RuleFiredEvent { + const outcome = typeof row.metadata.outcome === "string" ? row.metadata.outcome : ""; + const extraMetadata = { ...row.metadata }; + delete extraMetadata.outcome; + return { + ruleId, + targetKey: row.targetKey ?? "", + outcome, + occurredAt: row.createdAt, + ...(Object.keys(extraMetadata).length > 0 ? { metadata: extraMetadata } : {}), + }; +} + +// Mirrors src/review/signal-tracking-wire.ts's toHumanOverrideEvent — keep in sync with that adapter. +function toHumanOverrideEvent(ruleId: string, row: { targetKey: string | null; metadata: Record; createdAt: string }): HumanOverrideEvent { + const verdict = row.metadata.verdict === "reversed" ? "reversed" : "confirmed"; + const extraMetadata = { ...row.metadata }; + delete extraMetadata.verdict; + return { + ruleId, + targetKey: row.targetKey ?? "", + verdict, + occurredAt: row.createdAt, + ...(Object.keys(extraMetadata).length > 0 ? { metadata: extraMetadata } : {}), + }; +} + +function rowCreatedAt(row: D1Row): string { + return typeof row.created_at === "string" ? row.created_at : ""; +} + +function rowTargetKey(row: D1Row): string | null { + return typeof row.target_key === "string" ? row.target_key : null; +} + +function rowEventType(row: D1Row): string { + return typeof row.event_type === "string" ? row.event_type : ""; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + if (!args.ruleId || !args.output) { + console.error( + "Usage: tsx scripts/backtest-corpus-export.ts --rule-id --output [--remote] [--since-date ] [--db loopover]", + ); + process.exit(2); + } + + const firedType = `${RULE_FIRED_EVENT_TYPE_PREFIX}${args.ruleId}`; + const overrideType = `${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}${args.ruleId}`; + const sinceClause = args.sinceDate ? ` AND created_at >= ${sqlStringLiteral(args.sinceDate)}` : ""; + const sql = + `SELECT event_type, target_key, metadata_json, created_at FROM audit_events` + + ` WHERE (event_type = ${sqlStringLiteral(firedType)} OR event_type = ${sqlStringLiteral(overrideType)})` + + sinceClause + + ` ORDER BY created_at ASC`; + + const rows = d1Query(args.db, args.remote, sql); + const fired: RuleFiredEvent[] = []; + const overrides: HumanOverrideEvent[] = []; + for (const row of rows) { + const projected = { + targetKey: rowTargetKey(row), + metadata: parseMetadataJson(row.metadata_json), + createdAt: rowCreatedAt(row), + }; + const eventType = rowEventType(row); + if (eventType === firedType) fired.push(toRuleFiredEvent(args.ruleId, projected)); + else if (eventType === overrideType) overrides.push(toHumanOverrideEvent(args.ruleId, projected)); + } + + const cases: BacktestCase[] = buildBacktestCorpus(args.ruleId, fired, overrides); + const manifest = buildBacktestCorpusManifest(args.ruleId, cases, { + generatedAt: new Date().toISOString(), + source: args.remote ? "d1-remote" : "d1-local", + database: args.db, + incremental: Boolean(args.sinceDate), + ...(args.sinceDate ? { sinceDate: args.sinceDate } : {}), + }); + writeFileSync(args.output, `${JSON.stringify(manifest, null, 2)}\n`); + console.error(`exported ${manifest.caseCount} cases for rule ${args.ruleId} (checksum ${manifest.checksum.slice(0, 12)}…) → ${args.output}`); +} + +main(); diff --git a/test/unit/backtest-corpus-export-core.test.ts b/test/unit/backtest-corpus-export-core.test.ts new file mode 100644 index 0000000000..a1a4554a43 --- /dev/null +++ b/test/unit/backtest-corpus-export-core.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import type { BacktestCase } from "@loopover/engine"; +import { buildBacktestCorpusManifest, checksumCases } from "../../scripts/backtest-corpus-export-core.js"; + +function sampleCase(overrides: Partial = {}): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey: "owner/repo#1", + outcome: "block", + label: "confirmed", + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + ...overrides, + }; +} + +describe("backtest-corpus-export-core checksumCases (#8084)", () => { + it("is stable for the same input regardless of key order within each case object", () => { + const a: BacktestCase[] = [sampleCase({ outcome: "block", label: "confirmed" })]; + // Rebuild with a different enumeration order by spreading into a freshly keyed object. + const b: BacktestCase[] = [ + { + decidedAt: a[0]!.decidedAt, + firedAt: a[0]!.firedAt, + label: a[0]!.label, + outcome: a[0]!.outcome, + ruleId: a[0]!.ruleId, + targetKey: a[0]!.targetKey, + }, + ]; + expect(checksumCases(a)).toBe(checksumCases(b)); + expect(checksumCases(a)).toMatch(/^[0-9a-f]{64}$/); + }); + + it("changes when cases changes", () => { + expect(checksumCases([sampleCase({ label: "confirmed" })])).not.toBe(checksumCases([sampleCase({ label: "reversed" })])); + expect(checksumCases([sampleCase()])).not.toBe(checksumCases([sampleCase(), sampleCase({ targetKey: "owner/repo#2" })])); + }); +}); + +describe("backtest-corpus-export-core buildBacktestCorpusManifest (#8084)", () => { + it("caseCount matches cases.length and an empty corpus produces a valid manifest with caseCount 0", () => { + const empty = buildBacktestCorpusManifest("missing_linked_issue", []); + expect(empty.ruleId).toBe("missing_linked_issue"); + expect(empty.caseCount).toBe(0); + expect(empty.cases).toEqual([]); + expect(empty.checksum).toBe(checksumCases([])); + expect(empty.checksum).toMatch(/^[0-9a-f]{64}$/); + + const cases = [sampleCase(), sampleCase({ targetKey: "owner/repo#2", label: "reversed" })]; + const filled = buildBacktestCorpusManifest("missing_linked_issue", cases); + expect(filled.caseCount).toBe(2); + expect(filled.cases).toEqual(cases); + expect(filled.checksum).toBe(checksumCases(cases)); + }); + + it("spreads meta fields into the result without reading a clock", () => { + const manifest = buildBacktestCorpusManifest("rule_a", [sampleCase({ ruleId: "rule_a" })], { + generatedAt: "2026-07-22T12:00:00.000Z", + source: "d1-local", + }); + expect(manifest.generatedAt).toBe("2026-07-22T12:00:00.000Z"); + expect(manifest.source).toBe("d1-local"); + expect(manifest.ruleId).toBe("rule_a"); + expect(manifest.caseCount).toBe(1); + }); + + it("copies cases rather than retaining the caller's array reference", () => { + const cases = [sampleCase()]; + const manifest = buildBacktestCorpusManifest("missing_linked_issue", cases); + expect(manifest.cases).toEqual(cases); + expect(manifest.cases).not.toBe(cases); + }); +});