diff --git a/packages/loopover-engine/src/calibration/backtest-report.ts b/packages/loopover-engine/src/calibration/backtest-report.ts new file mode 100644 index 0000000000..2960692f1d --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-report.ts @@ -0,0 +1,72 @@ +// Markdown rendering for backtest score/comparison data (#8088, part of the #8082 rule-precision +// backtest epic). BacktestScoreReport (#8085) and BacktestComparison (#8086) are plain data; this is the +// human-readable "receipt" a maintainer (and, per the parent epic, eventually an advisory CI comment) +// reads directly -- a deterministic pure function producing stable Markdown, not ad-hoc console logging. +// +// SELF-CONTAINED, PURE: string in, string out -- no IO, no wall-clock reads, byte-identical output for +// byte-identical input, the same posture as the rest of this calibration directory. + +import type { BacktestComparison } from "./backtest-compare.js"; +import type { BacktestScoreReport } from "./backtest-score.js"; + +/** Render null precision/recall as the literal `N/A` -- never 0, the word null, or an empty cell, + * mirroring the null-is-not-zero discipline BacktestScoreReport itself establishes (#8085). */ +function formatAxisValue(value: number | null): string { + return value === null ? "N/A" : String(value); +} + +/** + * Render one {@link BacktestScoreReport} as a Markdown table: the rule ID, case count, all four + * confusion-matrix counts, and precision/recall (null rendered as `N/A`). + */ +export function renderBacktestScoreReport(report: BacktestScoreReport): string { + return [ + `### Backtest score: \`${report.ruleId}\``, + "", + "| Metric | Value |", + "| --- | --- |", + `| Cases scored | ${report.caseCount} |`, + `| True positives | ${report.truePositive} |`, + `| False positives | ${report.falsePositive} |`, + `| True negatives | ${report.trueNegative} |`, + `| False negatives | ${report.falseNegative} |`, + `| Precision | ${formatAxisValue(report.precision)} |`, + `| Recall | ${formatAxisValue(report.recall)} |`, + "", + ].join("\n"); +} + +/** + * Render one {@link BacktestComparison} as Markdown: regressed axes under a "Regressed" heading, + * improved axes under a visually separate "Improved" heading (a section with no axes is omitted + * entirely, so nothing ever reads as regressed when it isn't), and a closing verdict line. The + * `"regressed"` closing line contains the literal word `REGRESSED` and states the change should not be + * merged -- exact wording a future automated consumer (the follow-up CI wiring) detects by string match + * without re-implementing the comparison logic. + */ +export function renderBacktestComparison(comparison: BacktestComparison): string { + const lines: string[] = [`### Backtest comparison: \`${comparison.ruleId}\``, ""]; + if (comparison.regressedAxes.length > 0) { + lines.push("**Regressed**"); + for (const axis of comparison.regressedAxes) { + lines.push(`- ${axis}: ${formatAxisValue(comparison.baseline[axis])} → ${formatAxisValue(comparison.candidate[axis])}`); + } + lines.push(""); + } + if (comparison.improvedAxes.length > 0) { + lines.push("**Improved**"); + for (const axis of comparison.improvedAxes) { + lines.push(`- ${axis}: ${formatAxisValue(comparison.baseline[axis])} → ${formatAxisValue(comparison.candidate[axis])}`); + } + lines.push(""); + } + if (comparison.verdict === "regressed") { + lines.push("Verdict: REGRESSED — do not merge."); + } else if (comparison.verdict === "improved") { + lines.push("Verdict: improved — no axis regressed."); + } else { + lines.push("Verdict: unchanged — no comparable axis moved."); + } + lines.push(""); + return lines.join("\n"); +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 51ab2d1714..a36a7e8410 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -166,6 +166,7 @@ export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; export * from "./calibration/backtest-score.js"; export * from "./calibration/backtest-compare.js"; +export * from "./calibration/backtest-report.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-report.test.ts b/packages/loopover-engine/test/backtest-report.test.ts new file mode 100644 index 0000000000..0395a19cd9 --- /dev/null +++ b/packages/loopover-engine/test/backtest-report.test.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + renderBacktestComparison, + renderBacktestScoreReport, + type BacktestComparison, + type BacktestScoreReport, +} from "../dist/index.js"; + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "missing_linked_issue", + caseCount: 4, + truePositive: 1, + falsePositive: 1, + trueNegative: 1, + falseNegative: 1, + precision: 0.5, + recall: 0.5, + ...overrides, + }; +} + +function comparison(overrides: Partial = {}): BacktestComparison { + return { + ruleId: "missing_linked_issue", + baseline: report(), + candidate: report({ precision: 0.75 }), + regressedAxes: [], + improvedAxes: ["precision"], + verdict: "improved", + ...overrides, + }; +} + +test("barrel: the public entrypoint re-exports both backtest renderers (#8088)", () => { + assert.equal(typeof renderBacktestScoreReport, "function"); + assert.equal(typeof renderBacktestComparison, "function"); +}); + +test("renderBacktestScoreReport: renders every count and both non-null axes, snapshot-exact", () => { + const rendered = renderBacktestScoreReport(report()); + assert.equal( + rendered, + [ + "### Backtest score: `missing_linked_issue`", + "", + "| Metric | Value |", + "| --- | --- |", + "| Cases scored | 4 |", + "| True positives | 1 |", + "| False positives | 1 |", + "| True negatives | 1 |", + "| False negatives | 1 |", + "| Precision | 0.5 |", + "| Recall | 0.5 |", + "", + ].join("\n"), + ); +}); + +test("renderBacktestScoreReport: null precision/recall render as N/A, never 0 or the word null", () => { + const rendered = renderBacktestScoreReport(report({ precision: null, recall: null })); + assert.ok(rendered.includes("| Precision | N/A |")); + assert.ok(rendered.includes("| Recall | N/A |")); + assert.ok(!rendered.includes("null")); +}); + +test("renderBacktestComparison: a regressed verdict contains the literal REGRESSED and a do-not-merge line", () => { + const rendered = renderBacktestComparison( + comparison({ regressedAxes: ["recall"], improvedAxes: ["precision"], verdict: "regressed", candidate: report({ precision: 0.9, recall: 0.4 }) }), + ); + assert.ok(rendered.includes("REGRESSED")); + assert.ok(rendered.includes("do not merge")); + assert.ok(rendered.includes("**Regressed**")); + assert.ok(rendered.includes("- recall: 0.5 → 0.4")); +}); + +test("renderBacktestComparison: improved-only output claims no regressed axis", () => { + const rendered = renderBacktestComparison(comparison()); + assert.ok(rendered.includes("**Improved**")); + assert.ok(rendered.includes("- precision: 0.5 → 0.75")); + assert.ok(!rendered.includes("**Regressed**")); + assert.ok(rendered.includes("Verdict: improved")); +}); + +test("renderBacktestComparison / renderBacktestScoreReport: byte-identical output for identical input", () => { + assert.equal(renderBacktestScoreReport(report()), renderBacktestScoreReport(report())); + assert.equal(renderBacktestComparison(comparison()), renderBacktestComparison(comparison())); +}); diff --git a/test/unit/backtest-report.test.ts b/test/unit/backtest-report.test.ts new file mode 100644 index 0000000000..b443415537 --- /dev/null +++ b/test/unit/backtest-report.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; +import type { BacktestComparison } from "../../packages/loopover-engine/src/calibration/backtest-compare"; +import { renderBacktestComparison, renderBacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-report"; +import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score"; + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "missing_linked_issue", + caseCount: 4, + truePositive: 1, + falsePositive: 1, + trueNegative: 1, + falseNegative: 1, + precision: 0.5, + recall: 0.5, + ...overrides, + }; +} + +function comparison(overrides: Partial = {}): BacktestComparison { + return { + ruleId: "missing_linked_issue", + baseline: report(), + candidate: report({ precision: 0.75 }), + regressedAxes: [], + improvedAxes: ["precision"], + verdict: "improved", + ...overrides, + }; +} + +describe("renderBacktestScoreReport (#8088)", () => { + it("renders every count and both non-null axes, snapshot-exact", () => { + expect(renderBacktestScoreReport(report())).toBe( + [ + "### Backtest score: `missing_linked_issue`", + "", + "| Metric | Value |", + "| --- | --- |", + "| Cases scored | 4 |", + "| True positives | 1 |", + "| False positives | 1 |", + "| True negatives | 1 |", + "| False negatives | 1 |", + "| Precision | 0.5 |", + "| Recall | 0.5 |", + "", + ].join("\n"), + ); + }); + + it("renders null precision/recall as N/A — never 0, never the word null", () => { + const rendered = renderBacktestScoreReport(report({ precision: null, recall: null })); + expect(rendered).toContain("| Precision | N/A |"); + expect(rendered).toContain("| Recall | N/A |"); + expect(rendered).not.toContain("null"); + }); + + it("is byte-identical for identical input", () => { + expect(renderBacktestScoreReport(report())).toBe(renderBacktestScoreReport(report())); + }); +}); + +describe("renderBacktestComparison (#8088)", () => { + it("renders a regressed verdict with the literal REGRESSED, a do-not-merge line, and the regressed axis sectioned", () => { + const rendered = renderBacktestComparison( + comparison({ + regressedAxes: ["recall"], + improvedAxes: ["precision"], + verdict: "regressed", + candidate: report({ precision: 0.9, recall: 0.4 }), + }), + ); + expect(rendered).toContain("Verdict: REGRESSED — do not merge."); + expect(rendered).toContain("**Regressed**"); + expect(rendered).toContain("- recall: 0.5 → 0.4"); + expect(rendered).toContain("**Improved**"); + expect(rendered).toContain("- precision: 0.5 → 0.9"); + // The regressed axis never bleeds into the improved section and vice versa. + expect(rendered.indexOf("**Regressed**")).toBeLessThan(rendered.indexOf("- recall:")); + expect(rendered.indexOf("- recall:")).toBeLessThan(rendered.indexOf("**Improved**")); + }); + + it("renders improved-only output without claiming any regressed axis", () => { + const rendered = renderBacktestComparison(comparison()); + expect(rendered).toContain("**Improved**"); + expect(rendered).toContain("Verdict: improved — no axis regressed."); + expect(rendered).not.toContain("**Regressed**"); + }); + + it("renders an unchanged verdict with neither axis section", () => { + const rendered = renderBacktestComparison(comparison({ improvedAxes: [], verdict: "unchanged", candidate: report() })); + expect(rendered).toContain("Verdict: unchanged — no comparable axis moved."); + expect(rendered).not.toContain("**Regressed**"); + expect(rendered).not.toContain("**Improved**"); + }); + + it("renders N/A for a null axis endpoint inside a section line", () => { + const rendered = renderBacktestComparison( + comparison({ + baseline: report({ recall: 0.5 }), + candidate: report({ recall: null, precision: 0.75 }), + regressedAxes: [], + improvedAxes: ["precision"], + verdict: "improved", + }), + ); + expect(rendered).toContain("- precision: 0.5 → 0.75"); + }); + + it("is byte-identical for identical input", () => { + expect(renderBacktestComparison(comparison())).toBe(renderBacktestComparison(comparison())); + }); +});