feat(miner-selfimprove): render prediction-calibration Prometheus metrics

packages/gittensory-engine/src/index.ts

@@ -146,6 +146,13 @@ export {
   type MinerTelemetryOutcomeBucket,
   type NormalizedMinerTelemetryEvent,
 } from "./miner-telemetry.js";
+export {
+  MINER_PREDICTIONS_TOTAL,
+  MINER_PREDICTION_CORRECT_TOTAL,
+  MINER_PREDICTION_INCORRECT_TOTAL,
+  renderMinerPredictionMetrics,
+  type MinerPredictionMetricRow,
+} from "./miner-prediction-metrics.js";
 export {
   ATTEMPT_LOG_EVENT_TYPES,
   createAttemptLogBuffer,

packages/gittensory-engine/src/miner-prediction-metrics.ts

@@ -0,0 +1,74 @@
+// Miner prediction-calibration metrics (#4264). A pure Prometheus text-exposition renderer for the miner's own
+// predicted-gate accuracy, the miner-side counterpart to the server's src/selfhost/metrics.ts registry. It turns
+// prediction-ledger rows (packages/gittensory-miner/lib/prediction-ledger.js `readPredictions`) — optionally
+// joined with their realized outcome — into counters a future dashboard can scrape.
+//
+// Scoped as an on-demand RENDERER, not a live HTTP registry: gittensory-miner is a local CLI, not a daemon, so a
+// caller renders this to stdout for its own scrape/cron setup and reads the ledger itself (no data collection of
+// its own lives here — this stays a pure, side-effect-free function like the rest of gittensory-engine). It mirrors
+// the metric-naming (`gittensory_miner_*_total`) and HELP/TYPE/label conventions of src/selfhost/metrics.ts rather
+// than importing across the package boundary.
+//
+// Counters emitted:
+// - `gittensory_miner_predictions_total{conclusion="..."}` — predictions recorded, one series per predicted
+//   conclusion (e.g. merge/close/hold).
+// - `gittensory_miner_prediction_correct_total` — predictions whose realized outcome matched the prediction.
+// - `gittensory_miner_prediction_incorrect_total` — predictions whose realized outcome differed.
+// The correct/incorrect counters only move for rows carrying a resolved outcome; unresolved rows count toward
+// `predictions_total` only, so the surface is meaningful before outcome-pairing exists and grows once it does.
+
+export const MINER_PREDICTIONS_TOTAL = "gittensory_miner_predictions_total";
+export const MINER_PREDICTION_CORRECT_TOTAL = "gittensory_miner_prediction_correct_total";
+export const MINER_PREDICTION_INCORRECT_TOTAL = "gittensory_miner_prediction_incorrect_total";
+
+/** One prediction-ledger row for metrics: its predicted `conclusion`, plus an optional realized-outcome pairing
+ *  (`correct`: true = matched, false = differed, null/undefined = not yet resolved). */
+export type MinerPredictionMetricRow = {
+  conclusion: string;
+  correct?: boolean | null;
+};
+
+/** Mirror src/selfhost/metrics.ts:204 — HELP text escapes backslash and newline. */
+function escapeHelpText(help: string): string {
+  return help.replace(/\\/g, "\\\\").replace(/\n/g, "\\n");
+}
+
+/** Prometheus label-value escaping (backslash, double-quote, newline), a correctness-complete superset of
+ *  src/selfhost/metrics.ts:193's `"`-only escape so an arbitrary conclusion string can never break the line. */
+function escapeLabelValue(value: string): string {
+  return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
+}
+
+/**
+ * Render prediction-calibration counters as Prometheus text-exposition format. Pure and side-effect-free: a caller
+ * supplies the ledger rows (joined with any resolved outcomes) and prints the result. Deterministic — conclusion
+ * series are emitted in sorted order. Always emits HELP/TYPE for every counter, so the surface is well-formed even
+ * for an empty ledger.
+ */
+export function renderMinerPredictionMetrics(rows: readonly MinerPredictionMetricRow[]): string {
+  const totalByConclusion = new Map<string, number>();
+  let correct = 0;
+  let incorrect = 0;
+  for (const row of rows) {
+    totalByConclusion.set(row.conclusion, (totalByConclusion.get(row.conclusion) ?? 0) + 1);
+    if (row.correct === true) correct += 1;
+    else if (row.correct === false) incorrect += 1;
+  }
+
+  const lines: string[] = [];
+  lines.push(`# HELP ${MINER_PREDICTIONS_TOTAL} ${escapeHelpText("Gate-outcome predictions the miner has recorded, by predicted conclusion.")}`);
+  lines.push(`# TYPE ${MINER_PREDICTIONS_TOTAL} counter`);
+  for (const [conclusion, count] of [...totalByConclusion.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+    lines.push(`${MINER_PREDICTIONS_TOTAL}{conclusion="${escapeLabelValue(conclusion)}"} ${count}`);
+  }
+
+  lines.push(`# HELP ${MINER_PREDICTION_CORRECT_TOTAL} ${escapeHelpText("Predictions whose realized outcome matched the predicted conclusion.")}`);
+  lines.push(`# TYPE ${MINER_PREDICTION_CORRECT_TOTAL} counter`);
+  lines.push(`${MINER_PREDICTION_CORRECT_TOTAL} ${correct}`);
+
+  lines.push(`# HELP ${MINER_PREDICTION_INCORRECT_TOTAL} ${escapeHelpText("Predictions whose realized outcome differed from the predicted conclusion.")}`);
+  lines.push(`# TYPE ${MINER_PREDICTION_INCORRECT_TOTAL} counter`);
+  lines.push(`${MINER_PREDICTION_INCORRECT_TOTAL} ${incorrect}`);
+
+  return `${lines.join("\n")}\n`;
+}

test/unit/miner-prediction-metrics.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from "vitest";
+import {
+  MINER_PREDICTIONS_TOTAL,
+  MINER_PREDICTION_CORRECT_TOTAL,
+  MINER_PREDICTION_INCORRECT_TOTAL,
+  renderMinerPredictionMetrics,
+} from "../../packages/gittensory-engine/src/index";
+
+/** Parse `name{labels} value` / `name value` data lines out of an exposition string, keyed for easy assertions. */
+function dataLines(text: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const line of text.split("\n")) {
+    if (!line || line.startsWith("#")) continue;
+    const idx = line.lastIndexOf(" ");
+    out[line.slice(0, idx)] = line.slice(idx + 1);
+  }
+  return out;
+}
+
+describe("miner prediction-calibration metrics (#4264)", () => {
+  it("re-exports the renderer and metric-name constants from the engine barrel", () => {
+    expect(typeof renderMinerPredictionMetrics).toBe("function");
+    expect(MINER_PREDICTIONS_TOTAL).toBe("gittensory_miner_predictions_total");
+    expect(MINER_PREDICTION_CORRECT_TOTAL).toBe("gittensory_miner_prediction_correct_total");
+    expect(MINER_PREDICTION_INCORRECT_TOTAL).toBe("gittensory_miner_prediction_incorrect_total");
+  });
+
+  it("emits well-formed HELP/TYPE and zeroed counters for an empty ledger", () => {
+    const text = renderMinerPredictionMetrics([]);
+    expect(text.endsWith("\n")).toBe(true);
+    expect(text).toContain(`# HELP ${MINER_PREDICTIONS_TOTAL} `);
+    expect(text).toContain(`# TYPE ${MINER_PREDICTIONS_TOTAL} counter`);
+    // no predictions_total series when empty; correct/incorrect are single zeroed lines
+    expect(text).not.toContain(`${MINER_PREDICTIONS_TOTAL}{`);
+    expect(dataLines(text)).toEqual({
+      [MINER_PREDICTION_CORRECT_TOTAL]: "0",
+      [MINER_PREDICTION_INCORRECT_TOTAL]: "0",
+    });
+  });
+
+  it("counts predictions per conclusion in sorted order and ignores unresolved rows for correct/incorrect", () => {
+    const text = renderMinerPredictionMetrics([
+      { conclusion: "merge" },
+      { conclusion: "merge", correct: true },
+      { conclusion: "close", correct: false },
+      { conclusion: "hold" }, // unresolved: counts toward total only
+      { conclusion: "merge", correct: null }, // explicit unresolved
+    ]);
+    const d = dataLines(text);
+    expect(d[`${MINER_PREDICTIONS_TOTAL}{conclusion="merge"}`]).toBe("3");
+    expect(d[`${MINER_PREDICTIONS_TOTAL}{conclusion="close"}`]).toBe("1");
+    expect(d[`${MINER_PREDICTIONS_TOTAL}{conclusion="hold"}`]).toBe("1");
+    expect(d[MINER_PREDICTION_CORRECT_TOTAL]).toBe("1");
+    expect(d[MINER_PREDICTION_INCORRECT_TOTAL]).toBe("1");
+
+    // deterministic: conclusion series are alphabetically sorted (close, hold, merge)
+    const order = [...text.matchAll(/conclusion="([^"]+)"/g)].map((m) => m[1]);
+    expect(order).toEqual(["close", "hold", "merge"]);
+  });
+
+  it("escapes backslashes, quotes, and newlines in a conclusion label value", () => {
+    const text = renderMinerPredictionMetrics([{ conclusion: 'we"ird\\\nvalue' }]);
+    expect(text).toContain(`${MINER_PREDICTIONS_TOTAL}{conclusion="we\\"ird\\\\\\nvalue"} 1`);
+  });
+});