From be6f2df240ecc42306cd1191e41cc1e7baf9f922 Mon Sep 17 00:00:00 2001 From: cleanjunc Date: Wed, 22 Jul 2026 23:02:19 +0000 Subject: [PATCH] feat(engine): render backtest score and comparison reports as deterministic Markdown (#8088) --- .../src/calibration/backtest-report.ts | 67 +++++++++++++++ packages/loopover-engine/src/index.ts | 1 + .../test/backtest-report.test.ts | 86 +++++++++++++++++++ test/unit/backtest-report.test.ts | 79 +++++++++++++++++ 4 files changed, 233 insertions(+) create mode 100644 packages/loopover-engine/src/calibration/backtest-report.ts create mode 100644 packages/loopover-engine/test/backtest-report.test.ts create mode 100644 test/unit/backtest-report.test.ts diff --git a/packages/loopover-engine/src/calibration/backtest-report.ts b/packages/loopover-engine/src/calibration/backtest-report.ts new file mode 100644 index 0000000000..920cc95a9d --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-report.ts @@ -0,0 +1,67 @@ +// Markdown renderers for backtest results (#8088, parent epic #8082) -- the human-readable "receipt" a +// maintainer (and, per the epic's proposal, a future advisory CI comment) reads directly. Deterministic pure +// functions producing stable Markdown: byte-identical input -> byte-identical output, never ad-hoc logging. +// +// Pure, like everything in this module: string in, string out; no IO, no wall-clock reads. + +import type { BacktestComparison } from "./backtest-compare.js"; +import type { BacktestScoreReport } from "./backtest-score.js"; + +/** Render a null-able rate as its number, or the literal `N/A` -- never `0`, `null`, or an empty cell (the + * null-is-not-zero discipline BacktestScoreReport itself establishes). */ +function renderRate(value: number | null): string { + return value === null ? "N/A" : String(value); +} + +/** Render one score report as a Markdown table: rule ID, case count, all four confusion-matrix counts, and + * precision/recall (null rendered as `N/A`). */ +export function renderBacktestScoreReport(report: BacktestScoreReport): string { + return [ + `### Backtest score: \`${report.ruleId}\``, + "", + "| Metric | Value |", + "| --- | --- |", + `| Cases scored | ${report.caseCount} |`, + `| True positives | ${report.truePositive} |`, + `| False positives | ${report.falsePositive} |`, + `| True negatives | ${report.trueNegative} |`, + `| False negatives | ${report.falseNegative} |`, + `| Precision | ${renderRate(report.precision)} |`, + `| Recall | ${renderRate(report.recall)} |`, + "", + ].join("\n"); +} + +/** + * Render a comparison with clearly separated Regressed / Improved sections (an axis only ever appears under + * its own heading) and a closing verdict line. The regressed closing line contains the literal word + * `REGRESSED` and states the change should not be merged -- exact wording a future automated consumer can + * string-match without re-implementing the comparison. + */ +export function renderBacktestComparison(comparison: BacktestComparison): string { + const lines: string[] = [`### Backtest comparison: \`${comparison.ruleId}\``, ""]; + if (comparison.regressedAxes.length > 0) { + lines.push("**Regressed**", ""); + for (const axis of comparison.regressedAxes) { + // A listed axis is non-null on BOTH sides by compareBacktestScores's own exclusion rule. + lines.push(`- ${axis}: ${comparison.baseline[axis]} → ${comparison.candidate[axis]}`); + } + lines.push(""); + } + if (comparison.improvedAxes.length > 0) { + lines.push("**Improved**", ""); + for (const axis of comparison.improvedAxes) { + lines.push(`- ${axis}: ${comparison.baseline[axis]} → ${comparison.candidate[axis]}`); + } + lines.push(""); + } + if (comparison.verdict === "regressed") { + lines.push("Verdict: REGRESSED — do not merge (a regression on any axis outweighs improvement on another)."); + } else if (comparison.verdict === "improved") { + lines.push("Verdict: improved — no axis regressed and at least one improved."); + } else { + lines.push("Verdict: unchanged — no comparable axis moved in either direction."); + } + lines.push(""); + return lines.join("\n"); +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 51ab2d1714..a36a7e8410 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -166,6 +166,7 @@ export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; export * from "./calibration/backtest-score.js"; export * from "./calibration/backtest-compare.js"; +export * from "./calibration/backtest-report.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-report.test.ts b/packages/loopover-engine/test/backtest-report.test.ts new file mode 100644 index 0000000000..87a04c48a8 --- /dev/null +++ b/packages/loopover-engine/test/backtest-report.test.ts @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + compareBacktestScores, + renderBacktestComparison, + renderBacktestScoreReport, + type BacktestScoreReport, +} from "../dist/index.js"; + +// #8088: the Markdown "receipt" renderers. Null rates render as the literal `N/A` (never 0/null/empty), a +// regressed comparison's closing line contains the literal REGRESSED + do-not-merge wording, and both +// functions are byte-identically deterministic. + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "rule", + caseCount: 10, + truePositive: 4, + falsePositive: 1, + trueNegative: 4, + falseNegative: 1, + precision: 0.8, + recall: 0.8, + ...overrides, + }; +} + +test("renderBacktestScoreReport: exact snapshot for a non-null fixture", () => { + assert.equal( + renderBacktestScoreReport(report()), + [ + "### Backtest score: `rule`", + "", + "| Metric | Value |", + "| --- | --- |", + "| Cases scored | 10 |", + "| True positives | 4 |", + "| False positives | 1 |", + "| True negatives | 4 |", + "| False negatives | 1 |", + "| Precision | 0.8 |", + "| Recall | 0.8 |", + "", + ].join("\n"), + ); +}); + +test("renderBacktestScoreReport: null precision/recall render as the literal N/A, never 0 or null", () => { + const rendered = renderBacktestScoreReport(report({ precision: null, recall: null })); + assert.ok(rendered.includes("| Precision | N/A |")); + assert.ok(rendered.includes("| Recall | N/A |")); + assert.ok(!rendered.includes("| Precision | 0 |")); + assert.ok(!rendered.includes("null")); +}); + +test("renderBacktestComparison: a regressed verdict's closing line contains REGRESSED and do-not-merge wording", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.95, recall: 0.6 })); + const rendered = renderBacktestComparison(comparison); + assert.ok(rendered.includes("REGRESSED")); + assert.ok(rendered.includes("do not merge")); + // The regressed axis appears only under Regressed; the improved axis only under Improved. + const regressedSection = rendered.slice(rendered.indexOf("**Regressed**"), rendered.indexOf("**Improved**")); + assert.ok(regressedSection.includes("recall")); + assert.ok(!regressedSection.includes("- precision")); +}); + +test("renderBacktestComparison: an improved comparison renders no regressed axis claim", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.85 })); + const rendered = renderBacktestComparison(comparison); + assert.ok(rendered.includes("**Improved**")); + assert.ok(!rendered.includes("**Regressed**")); + assert.ok(rendered.includes("Verdict: improved")); +}); + +test("renderBacktestComparison: unchanged verdict states unchanged in words", () => { + const rendered = renderBacktestComparison(compareBacktestScores(report(), report())); + assert.ok(rendered.includes("Verdict: unchanged")); +}); + +test("both renderers are byte-identically deterministic for identical input", () => { + const scoreReport = report({ precision: null }); + assert.equal(renderBacktestScoreReport(scoreReport), renderBacktestScoreReport(scoreReport)); + const comparison = compareBacktestScores(report(), report({ recall: 0.9 })); + assert.equal(renderBacktestComparison(comparison), renderBacktestComparison(comparison)); +}); diff --git a/test/unit/backtest-report.test.ts b/test/unit/backtest-report.test.ts new file mode 100644 index 0000000000..5b5047094d --- /dev/null +++ b/test/unit/backtest-report.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +// Direct src-path imports — the coverage-twin pattern test/unit/backtest-corpus.test.ts established for this +// module (the engine's own node:test suite runs against dist, invisible to codecov/patch). +import { compareBacktestScores } from "../../packages/loopover-engine/src/calibration/backtest-compare.js"; +import { renderBacktestComparison, renderBacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-report.js"; +import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score.js"; + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "rule", + caseCount: 10, + truePositive: 4, + falsePositive: 1, + trueNegative: 4, + falseNegative: 1, + precision: 0.8, + recall: 0.8, + ...overrides, + }; +} + +describe("renderBacktestScoreReport (#8088)", () => { + it("renders the exact table for a non-null fixture (snapshot)", () => { + expect(renderBacktestScoreReport(report())).toBe( + [ + "### Backtest score: `rule`", + "", + "| Metric | Value |", + "| --- | --- |", + "| Cases scored | 10 |", + "| True positives | 4 |", + "| False positives | 1 |", + "| True negatives | 4 |", + "| False negatives | 1 |", + "| Precision | 0.8 |", + "| Recall | 0.8 |", + "", + ].join("\n"), + ); + }); + + it("renders null precision/recall as the literal N/A", () => { + const rendered = renderBacktestScoreReport(report({ precision: null, recall: null })); + expect(rendered).toContain("| Precision | N/A |"); + expect(rendered).toContain("| Recall | N/A |"); + expect(rendered).not.toContain("null"); + }); +}); + +describe("renderBacktestComparison (#8088)", () => { + it("keeps regressed and improved axes in visually separate sections and closes with REGRESSED — do not merge", () => { + const rendered = renderBacktestComparison(compareBacktestScores(report(), report({ precision: 0.95, recall: 0.6 }))); + expect(rendered).toContain("REGRESSED"); + expect(rendered).toContain("do not merge"); + const regressedSection = rendered.slice(rendered.indexOf("**Regressed**"), rendered.indexOf("**Improved**")); + expect(regressedSection).toContain("recall"); + expect(regressedSection).not.toContain("- precision"); + }); + + it("renders an improved comparison with no Regressed section at all", () => { + const rendered = renderBacktestComparison(compareBacktestScores(report(), report({ precision: 0.9, recall: 0.85 }))); + expect(rendered).not.toContain("**Regressed**"); + expect(rendered).toContain("Verdict: improved"); + }); + + it("omits a null-excluded axis from both sections entirely", () => { + const rendered = renderBacktestComparison(compareBacktestScores(report({ precision: null }), report({ precision: null, recall: 0.9 }))); + expect(rendered).not.toContain("precision"); + expect(rendered).toContain("- recall: 0.8 → 0.9"); + expect(rendered).toContain("Verdict: improved"); + }); + + it("states the unchanged verdict in words and is byte-identically deterministic", () => { + const comparison = compareBacktestScores(report(), report()); + const first = renderBacktestComparison(comparison); + expect(first).toContain("Verdict: unchanged"); + expect(renderBacktestComparison(comparison)).toBe(first); + }); +});