From b5b95e9f7b7dcce4f6bf7cb99b176a53d5155a32 Mon Sep 17 00:00:00 2001 From: cleanjunc Date: Sun, 12 Jul 2026 21:29:44 +0000 Subject: [PATCH] feat(miner): add a metrics command that renders prediction-calibration Prometheus text (#4838) --- .../gittensory-miner/bin/gittensory-miner.js | 7 ++ packages/gittensory-miner/lib/cli.js | 1 + .../gittensory-miner/lib/metrics-cli.d.ts | 9 ++ packages/gittensory-miner/lib/metrics-cli.js | 51 ++++++++ packages/gittensory-miner/package.json | 2 +- test/unit/miner-cli.test.ts | 1 + test/unit/miner-metrics-cli.test.ts | 115 ++++++++++++++++++ 7 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 packages/gittensory-miner/lib/metrics-cli.d.ts create mode 100644 packages/gittensory-miner/lib/metrics-cli.js create mode 100644 test/unit/miner-metrics-cli.test.ts diff --git a/packages/gittensory-miner/bin/gittensory-miner.js b/packages/gittensory-miner/bin/gittensory-miner.js index 39c1669c57..fc85d6b106 100755 --- a/packages/gittensory-miner/bin/gittensory-miner.js +++ b/packages/gittensory-miner/bin/gittensory-miner.js @@ -9,6 +9,7 @@ import { runLedgerCli } from "../lib/event-ledger-cli.js"; import { runLoop } from "../lib/loop-cli.js"; import { runManagePoll } from "../lib/manage-poll.js"; import { runManageStatus } from "../lib/manage-status.js"; +import { runMetrics } from "../lib/metrics-cli.js"; import { runPlanCli } from "../lib/plan-store-cli.js"; import { runClaimCli } from "../lib/claim-ledger-cli.js"; import { runQueueCli } from "../lib/portfolio-queue-cli.js"; @@ -42,6 +43,12 @@ if (cliArgs[0] === "doctor") { process.exit(runDoctor(cliArgs.slice(1))); } +// `metrics` is strictly local + offline like `status`/`doctor` (it reads only the local prediction ledger), so it +// is dispatched here, before the opportunistic npm-registry update check is ever started. +if (cliArgs[0] === "metrics") { + process.exit(runMetrics(cliArgs.slice(1))); +} + if (cliArgs[0] === "manage" && cliArgs[1] === "status") { process.exit(runManageStatus(cliArgs.slice(2))); } diff --git a/packages/gittensory-miner/lib/cli.js b/packages/gittensory-miner/lib/cli.js index d4e8c3838c..b303be043d 100644 --- a/packages/gittensory-miner/lib/cli.js +++ b/packages/gittensory-miner/lib/cli.js @@ -17,6 +17,7 @@ export function printHelp(input) { " gittensory-miner init [--json] [--verify-token] Bootstrap laptop-mode local SQLite state", " gittensory-miner status [--json] Show installed versions + local state paths", " gittensory-miner doctor [--json] Check this laptop is set up correctly", + " gittensory-miner metrics Print prediction-calibration counters in Prometheus text format", " gittensory-miner manage status [--json] Show managed PR rows from local portfolio + ledger", " gittensory-miner manage poll [--branch ] [--json]", " gittensory-miner discover [...] [--json]", diff --git a/packages/gittensory-miner/lib/metrics-cli.d.ts b/packages/gittensory-miner/lib/metrics-cli.d.ts new file mode 100644 index 0000000000..921b782d90 --- /dev/null +++ b/packages/gittensory-miner/lib/metrics-cli.d.ts @@ -0,0 +1,9 @@ +import type { MinerPredictionMetricRow } from "@jsonbored/gittensory-engine"; +import type { PredictionLedger } from "./prediction-ledger.js"; + +export function collectPredictionMetricRows(ledger: PredictionLedger): MinerPredictionMetricRow[]; + +export function runMetrics( + args: string[], + options?: { initPredictionLedger?: () => PredictionLedger }, +): number; diff --git a/packages/gittensory-miner/lib/metrics-cli.js b/packages/gittensory-miner/lib/metrics-cli.js new file mode 100644 index 0000000000..d9cc32366e --- /dev/null +++ b/packages/gittensory-miner/lib/metrics-cli.js @@ -0,0 +1,51 @@ +import { renderMinerPredictionMetrics } from "@jsonbored/gittensory-engine"; +import { initPredictionLedger } from "./prediction-ledger.js"; + +// `metrics` (#4838): render the miner's prediction-calibration counters as Prometheus text-exposition to stdout, +// for a scrape wrapper or cron redirect. The counters are produced by the engine's already-built +// renderMinerPredictionMetrics (packages/gittensory-engine/src/miner-prediction-metrics.ts) -- this command only +// reads the local prediction ledger and feeds it in, never touching the renderer itself. Strictly local + offline: +// no network, no writes. + +const METRICS_USAGE = "Usage: gittensory-miner metrics"; + +/** + * Project prediction-ledger rows onto the engine renderer's metric-row shape -- the predicted `conclusion` only. + * The realized-outcome pairing (`correct`) is intentionally left unset: the miner has no outcome-join yet, so the + * correct/incorrect counters stay zero and only `predictions_total{conclusion}` moves -- exactly how the renderer + * is designed to degrade before outcome-pairing exists (see its header comment). + */ +export function collectPredictionMetricRows(ledger) { + return ledger.readPredictions().map((entry) => ({ conclusion: entry.conclusion })); +} + +// Open the local prediction ledger (or a test-injected one) for the duration of `run`, closing it only when we +// opened it -- an injected ledger is owned by the caller. Mirrors event-ledger-cli.js's withEventLedger. +function withPredictionLedger(options, run) { + const ownsLedger = options.initPredictionLedger === undefined; + const ledger = (options.initPredictionLedger ?? initPredictionLedger)(); + try { + return run(ledger); + } finally { + if (ownsLedger) ledger.close(); + } +} + +export function runMetrics(args, options = {}) { + if (args.length > 0) { + console.error(METRICS_USAGE); + return 2; + } + + try { + return withPredictionLedger(options, (ledger) => { + // renderMinerPredictionMetrics returns a newline-terminated document; console.log re-adds the terminator, so + // trim it to emit exactly one trailing newline. + console.log(renderMinerPredictionMetrics(collectPredictionMetricRows(ledger)).trimEnd()); + return 0; + }); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + return 2; + } +} diff --git a/packages/gittensory-miner/package.json b/packages/gittensory-miner/package.json index 9b36ed41ef..b868ac8a6a 100644 --- a/packages/gittensory-miner/package.json +++ b/packages/gittensory-miner/package.json @@ -33,7 +33,7 @@ "expected-engine.version" ], "scripts": { - "build": "node --check bin/gittensory-miner.js && node --check bin/gittensory-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-run.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/gate-verdict-poller.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-open-pr.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" + "build": "node --check bin/gittensory-miner.js && node --check bin/gittensory-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-run.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/gate-verdict-poller.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-open-pr.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/metrics-cli.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" }, "dependencies": { "@jsonbored/gittensory-engine": "*", diff --git a/test/unit/miner-cli.test.ts b/test/unit/miner-cli.test.ts index 59b08107ff..e36d35781b 100644 --- a/test/unit/miner-cli.test.ts +++ b/test/unit/miner-cli.test.ts @@ -64,6 +64,7 @@ describe("gittensory-miner CLI helpers", () => { const text = log.mock.calls[0]?.[0]; expect(text).toContain("gittensory-miner --help"); expect(text).toContain("gittensory-miner version"); + expect(text).toContain("gittensory-miner metrics"); expect(text).toContain("--no-update-check"); }); diff --git a/test/unit/miner-metrics-cli.test.ts b/test/unit/miner-metrics-cli.test.ts new file mode 100644 index 0000000000..90423df7d2 --- /dev/null +++ b/test/unit/miner-metrics-cli.test.ts @@ -0,0 +1,115 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { initPredictionLedger } from "../../packages/gittensory-miner/lib/prediction-ledger.js"; +import { + collectPredictionMetricRows, + runMetrics, +} from "../../packages/gittensory-miner/lib/metrics-cli.js"; +import type { PredictionLedger } from "../../packages/gittensory-miner/lib/prediction-ledger.d.ts"; + +const roots: string[] = []; +const ledgers: Array<{ close(): void }> = []; + +function tempLedger(): PredictionLedger { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-metrics-cli-")); + roots.push(root); + const ledger = initPredictionLedger(join(root, "prediction-ledger.sqlite3")); + ledgers.push(ledger); + return ledger; +} + +function tempDbPath() { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-metrics-cli-")); + roots.push(root); + return join(root, "prediction-ledger.sqlite3"); +} + +function appendPrediction(ledger: PredictionLedger, targetId: number, conclusion: string) { + ledger.appendPrediction({ repoFullName: "acme/widgets", targetId, conclusion, pack: "gittensor", engineVersion: "0.2.0" }); +} + +afterEach(() => { + for (const ledger of ledgers.splice(0)) ledger.close(); + vi.restoreAllMocks(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("gittensory-miner metrics CLI (#4838)", () => { + it("collectPredictionMetricRows projects ledger rows onto the renderer's conclusion-only shape", () => { + const ledger = tempLedger(); + appendPrediction(ledger, 1, "merge"); + appendPrediction(ledger, 2, "close"); + expect(collectPredictionMetricRows(ledger)).toEqual([{ conclusion: "merge" }, { conclusion: "close" }]); + }); + + it("runMetrics renders prediction counters as Prometheus text and returns 0", () => { + const ledger = tempLedger(); + appendPrediction(ledger, 1, "merge"); + appendPrediction(ledger, 2, "close"); + appendPrediction(ledger, 3, "merge"); + + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runMetrics([], { initPredictionLedger: () => ledger })).toBe(0); + + const text = String(log.mock.calls[0]?.[0]); + expect(text).toContain("# TYPE gittensory_miner_predictions_total counter"); + // Series are emitted in sorted conclusion order, so "close" precedes "merge". + expect(text).toContain('gittensory_miner_predictions_total{conclusion="close"} 1'); + expect(text).toContain('gittensory_miner_predictions_total{conclusion="merge"} 2'); + // No outcome-join exists yet, so both the correct and incorrect counters stay zero. + expect(text).toContain("gittensory_miner_prediction_correct_total 0"); + expect(text).toContain("gittensory_miner_prediction_incorrect_total 0"); + // The output is a single, once-terminated document (no doubled trailing blank line). + expect(text.endsWith("\n")).toBe(false); + }); + + it("runMetrics opens and closes its own default ledger when none is injected", () => { + const dbPath = tempDbPath(); + const seed = initPredictionLedger(dbPath); + appendPrediction(seed, 1, "hold"); + seed.close(); + + const prev = process.env.GITTENSORY_MINER_PREDICTION_LEDGER_DB; + process.env.GITTENSORY_MINER_PREDICTION_LEDGER_DB = dbPath; + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + try { + expect(runMetrics([])).toBe(0); + } finally { + if (prev === undefined) delete process.env.GITTENSORY_MINER_PREDICTION_LEDGER_DB; + else process.env.GITTENSORY_MINER_PREDICTION_LEDGER_DB = prev; + } + expect(String(log.mock.calls[0]?.[0])).toContain('gittensory_miner_predictions_total{conclusion="hold"} 1'); + }); + + it("runMetrics rejects unexpected arguments with a usage error", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + expect(runMetrics(["--json"], { initPredictionLedger: () => tempLedger() })).toBe(2); + expect(error).toHaveBeenCalledWith("Usage: gittensory-miner metrics"); + }); + + it("runMetrics surfaces a thrown Error message and exits non-zero", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + expect( + runMetrics([], { + initPredictionLedger: () => { + throw new Error("prediction ledger is locked"); + }, + }), + ).toBe(2); + expect(error).toHaveBeenCalledWith("prediction ledger is locked"); + }); + + it("runMetrics stringifies a non-Error throw", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + expect( + runMetrics([], { + initPredictionLedger: () => { + throw "prediction-ledger-unavailable"; + }, + }), + ).toBe(2); + expect(error).toHaveBeenCalledWith("prediction-ledger-unavailable"); + }); +});