feat(engine): render backtest score and comparison reports as Markdown

packages/loopover-engine/src/calibration/backtest-report.ts

@@ -0,0 +1,72 @@
+// Markdown rendering for backtest score/comparison data (#8088, part of the #8082 rule-precision
+// backtest epic). BacktestScoreReport (#8085) and BacktestComparison (#8086) are plain data; this is the
+// human-readable "receipt" a maintainer (and, per the parent epic, eventually an advisory CI comment)
+// reads directly -- a deterministic pure function producing stable Markdown, not ad-hoc console logging.
+//
+// SELF-CONTAINED, PURE: string in, string out -- no IO, no wall-clock reads, byte-identical output for
+// byte-identical input, the same posture as the rest of this calibration directory.
+
+import type { BacktestComparison } from "./backtest-compare.js";
+import type { BacktestScoreReport } from "./backtest-score.js";
+
+/** Render null precision/recall as the literal `N/A` -- never 0, the word null, or an empty cell,
+ *  mirroring the null-is-not-zero discipline BacktestScoreReport itself establishes (#8085). */
+function formatAxisValue(value: number | null): string {
+  return value === null ? "N/A" : String(value);
+}
+
+/**
+ * Render one {@link BacktestScoreReport} as a Markdown table: the rule ID, case count, all four
+ * confusion-matrix counts, and precision/recall (null rendered as `N/A`).
+ */
+export function renderBacktestScoreReport(report: BacktestScoreReport): string {
+  return [
+    `### Backtest score: \`${report.ruleId}\``,
+    "",
+    "| Metric | Value |",
+    "| --- | --- |",
+    `| Cases scored | ${report.caseCount} |`,
+    `| True positives | ${report.truePositive} |`,
+    `| False positives | ${report.falsePositive} |`,
+    `| True negatives | ${report.trueNegative} |`,
+    `| False negatives | ${report.falseNegative} |`,
+    `| Precision | ${formatAxisValue(report.precision)} |`,
+    `| Recall | ${formatAxisValue(report.recall)} |`,
+    "",
+  ].join("\n");
+}
+
+/**
+ * Render one {@link BacktestComparison} as Markdown: regressed axes under a "Regressed" heading,
+ * improved axes under a visually separate "Improved" heading (a section with no axes is omitted
+ * entirely, so nothing ever reads as regressed when it isn't), and a closing verdict line. The
+ * `"regressed"` closing line contains the literal word `REGRESSED` and states the change should not be
+ * merged -- exact wording a future automated consumer (the follow-up CI wiring) detects by string match
+ * without re-implementing the comparison logic.
+ */
+export function renderBacktestComparison(comparison: BacktestComparison): string {
+  const lines: string[] = [`### Backtest comparison: \`${comparison.ruleId}\``, ""];
+  if (comparison.regressedAxes.length > 0) {
+    lines.push("**Regressed**");
+    for (const axis of comparison.regressedAxes) {
+      lines.push(`- ${axis}: ${formatAxisValue(comparison.baseline[axis])} → ${formatAxisValue(comparison.candidate[axis])}`);
+    }
+    lines.push("");
+  }
+  if (comparison.improvedAxes.length > 0) {
+    lines.push("**Improved**");
+    for (const axis of comparison.improvedAxes) {
+      lines.push(`- ${axis}: ${formatAxisValue(comparison.baseline[axis])} → ${formatAxisValue(comparison.candidate[axis])}`);
+    }
+    lines.push("");
+  }
+  if (comparison.verdict === "regressed") {
+    lines.push("Verdict: REGRESSED — do not merge.");
+  } else if (comparison.verdict === "improved") {
+    lines.push("Verdict: improved — no axis regressed.");
+  } else {
+    lines.push("Verdict: unchanged — no comparable axis moved.");
+  }
+  lines.push("");
+  return lines.join("\n");
+}

packages/loopover-engine/src/index.ts

@@ -166,6 +166,7 @@ export * from "./calibration/signal-tracking.js";
 export * from "./calibration/backtest-corpus.js";
 export * from "./calibration/backtest-score.js";
 export * from "./calibration/backtest-compare.js";
+export * from "./calibration/backtest-report.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/backtest-report.test.ts

@@ -0,0 +1,91 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+  renderBacktestComparison,
+  renderBacktestScoreReport,
+  type BacktestComparison,
+  type BacktestScoreReport,
+} from "../dist/index.js";
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "missing_linked_issue",
+    caseCount: 4,
+    truePositive: 1,
+    falsePositive: 1,
+    trueNegative: 1,
+    falseNegative: 1,
+    precision: 0.5,
+    recall: 0.5,
+    ...overrides,
+  };
+}
+
+function comparison(overrides: Partial<BacktestComparison> = {}): BacktestComparison {
+  return {
+    ruleId: "missing_linked_issue",
+    baseline: report(),
+    candidate: report({ precision: 0.75 }),
+    regressedAxes: [],
+    improvedAxes: ["precision"],
+    verdict: "improved",
+    ...overrides,
+  };
+}
+
+test("barrel: the public entrypoint re-exports both backtest renderers (#8088)", () => {
+  assert.equal(typeof renderBacktestScoreReport, "function");
+  assert.equal(typeof renderBacktestComparison, "function");
+});
+
+test("renderBacktestScoreReport: renders every count and both non-null axes, snapshot-exact", () => {
+  const rendered = renderBacktestScoreReport(report());
+  assert.equal(
+    rendered,
+    [
+      "### Backtest score: `missing_linked_issue`",
+      "",
+      "| Metric | Value |",
+      "| --- | --- |",
+      "| Cases scored | 4 |",
+      "| True positives | 1 |",
+      "| False positives | 1 |",
+      "| True negatives | 1 |",
+      "| False negatives | 1 |",
+      "| Precision | 0.5 |",
+      "| Recall | 0.5 |",
+      "",
+    ].join("\n"),
+  );
+});
+
+test("renderBacktestScoreReport: null precision/recall render as N/A, never 0 or the word null", () => {
+  const rendered = renderBacktestScoreReport(report({ precision: null, recall: null }));
+  assert.ok(rendered.includes("| Precision | N/A |"));
+  assert.ok(rendered.includes("| Recall | N/A |"));
+  assert.ok(!rendered.includes("null"));
+});
+
+test("renderBacktestComparison: a regressed verdict contains the literal REGRESSED and a do-not-merge line", () => {
+  const rendered = renderBacktestComparison(
+    comparison({ regressedAxes: ["recall"], improvedAxes: ["precision"], verdict: "regressed", candidate: report({ precision: 0.9, recall: 0.4 }) }),
+  );
+  assert.ok(rendered.includes("REGRESSED"));
+  assert.ok(rendered.includes("do not merge"));
+  assert.ok(rendered.includes("**Regressed**"));
+  assert.ok(rendered.includes("- recall: 0.5 → 0.4"));
+});
+
+test("renderBacktestComparison: improved-only output claims no regressed axis", () => {
+  const rendered = renderBacktestComparison(comparison());
+  assert.ok(rendered.includes("**Improved**"));
+  assert.ok(rendered.includes("- precision: 0.5 → 0.75"));
+  assert.ok(!rendered.includes("**Regressed**"));
+  assert.ok(rendered.includes("Verdict: improved"));
+});
+
+test("renderBacktestComparison / renderBacktestScoreReport: byte-identical output for identical input", () => {
+  assert.equal(renderBacktestScoreReport(report()), renderBacktestScoreReport(report()));
+  assert.equal(renderBacktestComparison(comparison()), renderBacktestComparison(comparison()));
+});

test/unit/backtest-report.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, it } from "vitest";
+import type { BacktestComparison } from "../../packages/loopover-engine/src/calibration/backtest-compare";
+import { renderBacktestComparison, renderBacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-report";
+import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score";
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "missing_linked_issue",
+    caseCount: 4,
+    truePositive: 1,
+    falsePositive: 1,
+    trueNegative: 1,
+    falseNegative: 1,
+    precision: 0.5,
+    recall: 0.5,
+    ...overrides,
+  };
+}
+
+function comparison(overrides: Partial<BacktestComparison> = {}): BacktestComparison {
+  return {
+    ruleId: "missing_linked_issue",
+    baseline: report(),
+    candidate: report({ precision: 0.75 }),
+    regressedAxes: [],
+    improvedAxes: ["precision"],
+    verdict: "improved",
+    ...overrides,
+  };
+}
+
+describe("renderBacktestScoreReport (#8088)", () => {
+  it("renders every count and both non-null axes, snapshot-exact", () => {
+    expect(renderBacktestScoreReport(report())).toBe(
+      [
+        "### Backtest score: `missing_linked_issue`",
+        "",
+        "| Metric | Value |",
+        "| --- | --- |",
+        "| Cases scored | 4 |",
+        "| True positives | 1 |",
+        "| False positives | 1 |",
+        "| True negatives | 1 |",
+        "| False negatives | 1 |",
+        "| Precision | 0.5 |",
+        "| Recall | 0.5 |",
+        "",
+      ].join("\n"),
+    );
+  });
+
+  it("renders null precision/recall as N/A — never 0, never the word null", () => {
+    const rendered = renderBacktestScoreReport(report({ precision: null, recall: null }));
+    expect(rendered).toContain("| Precision | N/A |");
+    expect(rendered).toContain("| Recall | N/A |");
+    expect(rendered).not.toContain("null");
+  });
+
+  it("is byte-identical for identical input", () => {
+    expect(renderBacktestScoreReport(report())).toBe(renderBacktestScoreReport(report()));
+  });
+});
+
+describe("renderBacktestComparison (#8088)", () => {
+  it("renders a regressed verdict with the literal REGRESSED, a do-not-merge line, and the regressed axis sectioned", () => {
+    const rendered = renderBacktestComparison(
+      comparison({
+        regressedAxes: ["recall"],
+        improvedAxes: ["precision"],
+        verdict: "regressed",
+        candidate: report({ precision: 0.9, recall: 0.4 }),
+      }),
+    );
+    expect(rendered).toContain("Verdict: REGRESSED — do not merge.");
+    expect(rendered).toContain("**Regressed**");
+    expect(rendered).toContain("- recall: 0.5 → 0.4");
+    expect(rendered).toContain("**Improved**");
+    expect(rendered).toContain("- precision: 0.5 → 0.9");
+    // The regressed axis never bleeds into the improved section and vice versa.
+    expect(rendered.indexOf("**Regressed**")).toBeLessThan(rendered.indexOf("- recall:"));
+    expect(rendered.indexOf("- recall:")).toBeLessThan(rendered.indexOf("**Improved**"));
+  });
+
+  it("renders improved-only output without claiming any regressed axis", () => {
+    const rendered = renderBacktestComparison(comparison());
+    expect(rendered).toContain("**Improved**");
+    expect(rendered).toContain("Verdict: improved — no axis regressed.");
+    expect(rendered).not.toContain("**Regressed**");
+  });
+
+  it("renders an unchanged verdict with neither axis section", () => {
+    const rendered = renderBacktestComparison(comparison({ improvedAxes: [], verdict: "unchanged", candidate: report() }));
+    expect(rendered).toContain("Verdict: unchanged — no comparable axis moved.");
+    expect(rendered).not.toContain("**Regressed**");
+    expect(rendered).not.toContain("**Improved**");
+  });
+
+  it("renders N/A for a null axis endpoint inside a section line", () => {
+    const rendered = renderBacktestComparison(
+      comparison({
+        baseline: report({ recall: 0.5 }),
+        candidate: report({ recall: null, precision: 0.75 }),
+        regressedAxes: [],
+        improvedAxes: ["precision"],
+        verdict: "improved",
+      }),
+    );
+    expect(rendered).toContain("- precision: 0.5 → 0.75");
+  });
+
+  it("is byte-identical for identical input", () => {
+    expect(renderBacktestComparison(comparison())).toBe(renderBacktestComparison(comparison()));
+  });
+});