feat(miner): add a metrics command for the prediction Prometheus renderer

migrations/0147_predicted_gate_calibration_ledger_login_lower_idx.sql

@@ -0,0 +1,5 @@
+-- Keep the contributor-triggered calibration read indexed after login casing canonicalization (#2349).
+-- SQLite/D1 cannot use the plain (login, created_at) index for WHERE lower(login) = ?, so this matching
+-- expression index preserves case-insensitive lookup semantics without scanning the insert-only ledger.
+CREATE INDEX IF NOT EXISTS predicted_gate_calibration_ledger_login_lower_idx
+  ON predicted_gate_calibration_ledger(lower(login), created_at);

packages/gittensory-miner/bin/gittensory-miner.js

@@ -6,6 +6,7 @@ import { runDiscover } from "../lib/discover-cli.js";
 import { runFeasibilityCli } from "../lib/feasibility-cli.js";
 import { runGovernorCli } from "../lib/governor-ledger-cli.js";
 import { runLedgerCli } from "../lib/event-ledger-cli.js";
+import { runMetricsCli } from "../lib/prediction-metrics-cli.js";
 import { runLoop } from "../lib/loop-cli.js";
 import { runManagePoll } from "../lib/manage-poll.js";
 import { runManageStatus } from "../lib/manage-status.js";
@@ -62,6 +63,10 @@ if (cliArgs[0] === "ledger") {
   process.exit(runLedgerCli(cliArgs[1], cliArgs.slice(2)));
 }
 
+if (cliArgs[0] === "metrics") {
+  process.exit(runMetricsCli(cliArgs.slice(1)));
+}
+
 if (cliArgs[0] === "plan") {
   process.exit(runPlanCli(cliArgs[1], cliArgs.slice(2)));
 }

packages/gittensory-miner/lib/cli.js

@@ -36,6 +36,7 @@ export function printHelp(input) {
       "  gittensory-miner plan list [--status pending|running|completed|failed] [--json]",
       "  gittensory-miner plan show <planId> [--json]",
       "  gittensory-miner governor list [--repo <owner/repo>] [--type allowed|denied|throttled|kill_switch] [--json]",
+      "  gittensory-miner metrics                                      Print prediction-accuracy counters as Prometheus text",
       "  gittensory-miner feasibility <claimStatus> <duplicateClusterRisk> <issueStatus> [--not-found] [--json]",
       "  gittensory-miner hooks check --tool <name> --input <json> [--json]",
       "  gittensory-miner state get <owner/repo> [--json]",

packages/gittensory-miner/lib/prediction-metrics-cli.d.ts

@@ -0,0 +1 @@
+export function runMetricsCli(args?: string[], env?: Record<string, string | undefined>): number;

packages/gittensory-miner/lib/prediction-metrics-cli.js

@@ -0,0 +1,72 @@
+// `gittensory-miner metrics` (#4838): wire the already-built pure Prometheus renderer (gittensory-engine's
+// renderMinerPredictionMetrics, designed for cron/scrape use but previously never called) into a real command.
+// Reads the local prediction ledger, pairs each prediction with its realized PR outcome (event-ledger pr_outcome
+// events) to mark it correct/incorrect, and prints the renderer's Prometheus text-exposition output to stdout for
+// a scrape wrapper or cron redirect. Read-only; does not modify the renderer — it is already correct.
+import { renderMinerPredictionMetrics } from "@jsonbored/gittensory-engine";
+import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js";
+import { MINER_PR_OUTCOME_EVENT } from "./pr-outcome.js";
+import { initPredictionLedger, resolvePredictionLedgerDbPath } from "./prediction-ledger.js";
+
+const METRICS_USAGE = "Usage: gittensory-miner metrics";
+
+/** Normalize a predicted conclusion or a realized outcome decision to a shared vocabulary so the two can be
+ *  compared: `merged` → "merge", `closed` → "close", anything else lower-cased and trimmed. Both call sites pass a
+ *  guaranteed string (the ledger's `conclusion` is NOT NULL; a decision is `typeof`-checked before this is called). */
+function normalizeDecision(value) {
+  const text = value.trim().toLowerCase();
+  if (text === "merged") return "merge";
+  if (text === "closed") return "close";
+  return text;
+}
+
+/** Reduce the append-only pr_outcome event stream to the latest realized decision per `${repoFullName}:${targetId}`.
+ *  Non-outcome events and malformed payloads are skipped. */
+function latestOutcomeByKey(events) {
+  const latest = new Map();
+  for (const event of events) {
+    if (event?.type !== MINER_PR_OUTCOME_EVENT) continue;
+    const payload = event.payload;
+    if (!payload || !Number.isInteger(payload.prNumber) || typeof payload.decision !== "string") continue;
+    latest.set(`${event.repoFullName}:${payload.prNumber}`, normalizeDecision(payload.decision));
+  }
+  return latest;
+}
+
+/** Map prediction rows to the renderer's metric-row shape, marking `correct` only for predictions whose target has
+ *  a realized outcome (`null` leaves the row counted toward predictions_total but not correct/incorrect). */
+function toMetricRows(predictions, outcomesByKey) {
+  return predictions.map((prediction) => {
+    const outcome = outcomesByKey.get(`${prediction.repoFullName}:${prediction.targetId}`);
+    const correct = outcome === undefined ? null : normalizeDecision(prediction.conclusion) === outcome;
+    return { conclusion: prediction.conclusion, correct };
+  });
+}
+
+/**
+ * Run `gittensory-miner metrics`. Reads the prediction ledger + realized pr_outcome events, pairs them, and writes
+ * the existing renderer's Prometheus text-exposition output to stdout. Returns the process exit code: 0 on success,
+ * 1 on an unknown option.
+ * @param {string[]} [args]
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {number}
+ */
+export function runMetricsCli(args = [], env = process.env) {
+  const unknown = args.find((token) => token.startsWith("-"));
+  if (unknown) {
+    console.error(`Unknown option: ${unknown}. ${METRICS_USAGE}`);
+    return 1;
+  }
+
+  const predictionStore = initPredictionLedger(resolvePredictionLedgerDbPath(env));
+  const eventLedger = initEventLedger(resolveEventLedgerDbPath(env));
+  try {
+    const outcomesByKey = latestOutcomeByKey(eventLedger.readEvents());
+    const rows = toMetricRows(predictionStore.readPredictions(), outcomesByKey);
+    process.stdout.write(renderMinerPredictionMetrics(rows));
+    return 0;
+  } finally {
+    predictionStore.close();
+    eventLedger.close();
+  }
+}

test/unit/miner-prediction-metrics-cli.test.ts

@@ -0,0 +1,113 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("@jsonbored/gittensory-engine", async () => {
+  return import("../../packages/gittensory-engine/src/index");
+});
+
+import { initEventLedger, resolveEventLedgerDbPath } from "../../packages/gittensory-miner/lib/event-ledger.js";
+import {
+  initPredictionLedger,
+  resolvePredictionLedgerDbPath,
+} from "../../packages/gittensory-miner/lib/prediction-ledger.js";
+import { runMetricsCli } from "../../packages/gittensory-miner/lib/prediction-metrics-cli.js";
+
+const tempDirs: string[] = [];
+afterEach(() => {
+  vi.restoreAllMocks();
+  for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function envForTempStores(): Record<string, string | undefined> {
+  const dir = mkdtempSync(join(tmpdir(), "miner-metrics-cli-"));
+  tempDirs.push(dir);
+  return { GITTENSORY_MINER_CONFIG_DIR: dir };
+}
+
+function seedPrediction(env: Record<string, string | undefined>, targetId: number, conclusion: string) {
+  const store = initPredictionLedger(resolvePredictionLedgerDbPath(env));
+  store.appendPrediction({
+    repoFullName: "acme/widgets",
+    targetId,
+    conclusion,
+    pack: "oss",
+    readinessScore: 90,
+    blockerCodes: [],
+    warningCodes: [],
+    engineVersion: "1.0.0",
+  });
+  store.close();
+}
+
+function seedEvent(env: Record<string, string | undefined>, payload: Record<string, unknown>, type = "pr_outcome") {
+  const ledger = initEventLedger(resolveEventLedgerDbPath(env));
+  ledger.appendEvent({ type, repoFullName: "acme/widgets", payload });
+  ledger.close();
+}
+
+function captureStdout(): { text: () => string } {
+  const chunks: string[] = [];
+  vi.spyOn(process.stdout, "write").mockImplementation((chunk: unknown) => {
+    chunks.push(String(chunk));
+    return true;
+  });
+  return { text: () => chunks.join("") };
+}
+
+describe("gittensory-miner metrics CLI (#4838)", () => {
+  it("renders prediction counters and pairs realized outcomes into correct/incorrect", () => {
+    const env = envForTempStores();
+    seedPrediction(env, 1, "merge");
+    seedPrediction(env, 2, "close");
+    seedPrediction(env, 3, "hold"); // no realized outcome ⇒ counts to predictions_total only
+    seedEvent(env, { prNumber: 1, decision: "merged" }); // predicted merge, realized merge ⇒ correct
+    seedEvent(env, { prNumber: 2, decision: "merged" }); // predicted close, realized merge ⇒ incorrect
+    const out = captureStdout();
+
+    expect(runMetricsCli([], env)).toBe(0);
+    const text = out.text();
+    expect(text).toContain("# HELP gittensory_miner_predictions_total");
+    expect(text).toContain("# TYPE gittensory_miner_predictions_total counter");
+    expect(text).toContain('gittensory_miner_predictions_total{conclusion="close"} 1');
+    expect(text).toContain('gittensory_miner_predictions_total{conclusion="hold"} 1');
+    expect(text).toContain('gittensory_miner_predictions_total{conclusion="merge"} 1');
+    expect(text).toContain("gittensory_miner_prediction_correct_total 1");
+    expect(text).toContain("gittensory_miner_prediction_incorrect_total 1");
+  });
+
+  it("uses the latest outcome per PR and ignores non-outcome / malformed events", () => {
+    const env = envForTempStores();
+    seedPrediction(env, 5, "merge");
+    seedEvent(env, { prNumber: 5, decision: "closed" }); // earlier, superseded
+    seedEvent(env, { prNumber: 5, decision: "merged" }); // latest wins ⇒ correct
+    seedEvent(env, { note: "not an outcome" }, "some_other_event"); // wrong type ⇒ ignored
+    seedEvent(env, { prNumber: "bad", decision: "merged" }); // malformed prNumber ⇒ ignored
+    seedEvent(env, { prNumber: 9 }); // missing decision ⇒ ignored
+    const out = captureStdout();
+
+    expect(runMetricsCli([], env)).toBe(0);
+    const text = out.text();
+    expect(text).toContain("gittensory_miner_prediction_correct_total 1");
+    expect(text).toContain("gittensory_miner_prediction_incorrect_total 0");
+  });
+
+  it("emits a well-formed empty surface when the ledgers are empty", () => {
+    const env = envForTempStores();
+    const out = captureStdout();
+
+    expect(runMetricsCli([], env)).toBe(0);
+    const text = out.text();
+    expect(text).toContain("# TYPE gittensory_miner_predictions_total counter");
+    expect(text).toContain("gittensory_miner_prediction_correct_total 0");
+    expect(text).toContain("gittensory_miner_prediction_incorrect_total 0");
+    expect(text).not.toContain("predictions_total{"); // no series without any predictions
+  });
+
+  it("rejects an unknown option with exit code 1", () => {
+    const err = vi.spyOn(console, "error").mockImplementation(() => {});
+    expect(runMetricsCli(["--bogus"], envForTempStores())).toBe(1);
+    expect(String(err.mock.calls[0]?.[0])).toContain("Unknown option");
+  });
+});

test/unit/predicted-gate-calibration-ledger.test.ts

@@ -234,6 +234,26 @@ describe("computeContributorCalibration — per-login calibration read (#2349)",
     expect(await computeContributorCalibration(env, "someone-else")).toEqual({ sampleSize: 1, agreementRate: 1 });
   });
 
+  it("uses the matching lower(login) expression index for canonicalized calibration lookups", async () => {
+    const env = createTestEnv();
+
+    const idx = await env.DB.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name = ?")
+      .bind("predicted_gate_calibration_ledger_login_lower_idx")
+      .first<{ name: string }>();
+    expect(idx?.name).toBe("predicted_gate_calibration_ledger_login_lower_idx");
+
+    const plan = await env.DB.prepare(
+      `EXPLAIN QUERY PLAN SELECT COUNT(*) AS sampleSize, COALESCE(AVG(agreed), 0) AS agreementRate
+         FROM predicted_gate_calibration_ledger
+        WHERE lower(login) = ?`,
+    )
+      .bind("octocat")
+      .all<{ detail: string }>();
+    const detail = (plan.results ?? []).map((row) => row.detail).join(" ");
+    expect(detail).toContain("predicted_gate_calibration_ledger_login_lower_idx");
+    expect(detail).not.toContain("SCAN predicted_gate_calibration_ledger");
+  });
+
   it("aggregates across ALL of a login's history regardless of which repo each pairing came from", async () => {
     const env = createTestEnv();
     await seedLedgerRow(env, { login: "octocat", project: "owner/repo-a", pullNumber: 1, agreed: true });