diff --git a/packages/gittensory-miner/lib/cli.js b/packages/gittensory-miner/lib/cli.js index abb7d4c4b6..c1722c6a5b 100644 --- a/packages/gittensory-miner/lib/cli.js +++ b/packages/gittensory-miner/lib/cli.js @@ -48,6 +48,7 @@ export function printHelp(input) { " gittensory-miner governor pause [--reason ] [--dry-run] [--json] Stop the loop before its next cycle", " gittensory-miner governor resume [--dry-run] [--json] Let a paused loop continue", " gittensory-miner governor status [--json] Show whether the governor is paused", + " gittensory-miner governor metrics Print governor rate-limit/cap-usage counters in Prometheus text format", " gittensory-miner calibration [--json] Report predicted-vs-realized gate accuracy", " gittensory-miner feasibility [--not-found] [--json]", " gittensory-miner hooks check --tool --input [--json]", diff --git a/packages/gittensory-miner/lib/governor-ledger-cli.d.ts b/packages/gittensory-miner/lib/governor-ledger-cli.d.ts index a9263ee7db..ed1e2e1b8d 100644 --- a/packages/gittensory-miner/lib/governor-ledger-cli.d.ts +++ b/packages/gittensory-miner/lib/governor-ledger-cli.d.ts @@ -28,5 +28,5 @@ export function runGovernorList( export function runGovernorCli( subcommand: string | undefined, args: string[], - options?: { initGovernorLedger?: () => GovernorLedger } & GovernorPauseCliOptions, + options?: { initGovernorLedger?: () => GovernorLedger; nowMs?: number } & GovernorPauseCliOptions, ): Promise; diff --git a/packages/gittensory-miner/lib/governor-ledger-cli.js b/packages/gittensory-miner/lib/governor-ledger-cli.js index 6f1a219f9d..872654443d 100644 --- a/packages/gittensory-miner/lib/governor-ledger-cli.js +++ b/packages/gittensory-miner/lib/governor-ledger-cli.js @@ -1,4 +1,5 @@ import { runGovernorPause, runGovernorResume, runGovernorStatus } from "./governor-pause-cli.js"; +import { runGovernorMetrics } from "./governor-metrics-cli.js"; /** Must match `GOVERNOR_LEDGER_EVENT_TYPES` in `@loopover/engine`. */ import { argsWantJson, describeCliError, reportCliFailure } from "./cli-error.js"; @@ -18,6 +19,7 @@ const GOVERNOR_SUBCOMMAND_USAGE = [ " gittensory-miner governor pause [--reason ] [--json]", " gittensory-miner governor resume [--json]", " gittensory-miner governor status [--json]", + " gittensory-miner governor metrics", ].join("\n"); function parseRepoArg(value, usage) { @@ -148,6 +150,7 @@ export async function runGovernorCli(subcommand, args, options = {}) { if (subcommand === "pause") return runGovernorPause(args, options); if (subcommand === "resume") return runGovernorResume(args, options); if (subcommand === "status") return runGovernorStatus(args, options); + if (subcommand === "metrics") return runGovernorMetrics(args, options); return reportCliFailure( argsWantJson(args), `Unknown governor subcommand: ${subcommand ?? ""}.\n${GOVERNOR_SUBCOMMAND_USAGE}`, diff --git a/packages/gittensory-miner/lib/governor-metrics-cli.d.ts b/packages/gittensory-miner/lib/governor-metrics-cli.d.ts new file mode 100644 index 0000000000..c12edd72c2 --- /dev/null +++ b/packages/gittensory-miner/lib/governor-metrics-cli.d.ts @@ -0,0 +1,16 @@ +import type { GovernorCapUsage } from "@loopover/engine"; +import type { GovernorRateLimitState, GovernorState } from "./governor-state.js"; + +export const GOVERNOR_RATE_LIMIT_REMAINING_RATIO: string; +export const GOVERNOR_CAP_USAGE_RATIO: string; + +export function renderGovernorMetrics( + rateLimitState: GovernorRateLimitState, + capUsage: GovernorCapUsage, + nowMs: number, +): string; + +export function runGovernorMetrics( + args: string[], + options?: { openGovernorState?: () => GovernorState; nowMs?: number }, +): Promise; diff --git a/packages/gittensory-miner/lib/governor-metrics-cli.js b/packages/gittensory-miner/lib/governor-metrics-cli.js new file mode 100644 index 0000000000..9d36df6d18 --- /dev/null +++ b/packages/gittensory-miner/lib/governor-metrics-cli.js @@ -0,0 +1,171 @@ +import { + DEFAULT_AMS_POLICY_SPEC, + DEFAULT_WRITE_RATE_LIMIT_POLICIES, + evaluateGovernorCaps, + evaluateLocalRateLimit, +} from "@loopover/engine"; +import { openGovernorState } from "./governor-state.js"; +import { argsWantJson, describeCliError, reportCliFailure } from "./cli-error.js"; + +// `governor metrics` (#5187): render the governor's persisted rate-limit + cap-usage state (#5134, +// governor-state.js) as Prometheus text-exposition, so an operator's Alertmanager can page on rate-limit/ +// budget pressure without hand-rolling a scrape. Strictly read-only, mirroring queue-cli.js's `queue metrics` +// (#5186) and event-ledger-cli.js's `ledger metrics` (#4841): opens the local governor-state store, composes +// its EXISTING loadRateLimitState()/loadCapUsage() with the engine's already-exported PURE calculators +// (evaluateLocalRateLimit, evaluateGovernorCaps) against the SAME defaults the production loop (loop-cli.js) +// already falls back to when no `.gittensory-ams.yml` override is configured (DEFAULT_WRITE_RATE_LIMIT_POLICIES, +// DEFAULT_AMS_POLICY_SPEC.capLimits) -- it never invents a threshold of its own, and it does not gate, retry, +// mutate, or otherwise touch governor decision logic (governor-chokepoint.js/governor-chokepoint-persisted.js +// are completely untouched by this file). +// +// capLimits is intentionally NOT read per-repo: governor-state.js's capUsage row is a single global scalar (a +// run-scoped cumulative counter, not indexed by repo -- see governor-state.js's own header comment), so a +// per-repo capLimits override from a resolved `.gittensory-miner.yml` has no matching per-repo usage row to +// pair it with here. Using the fleet-wide DEFAULT_AMS_POLICY_SPEC.capLimits is the same approximation +// loop-cli.js itself already makes for any repo without its own override. + +const GOVERNOR_METRICS_USAGE = "Usage: gittensory-miner governor metrics"; + +export const GOVERNOR_RATE_LIMIT_REMAINING_RATIO = "gittensory_miner_governor_rate_limit_remaining_ratio"; +export const GOVERNOR_CAP_USAGE_RATIO = "gittensory_miner_governor_cap_usage_ratio"; + +/** HELP-text escaping — backslash + newline (mirrors miner-prediction-metrics.ts's escapeHelpText). */ +function escapeMetricsHelpText(help) { + return help.replace(/\\/g, "\\\\").replace(/\n/g, "\\n"); +} + +/** Prometheus label-value escaping — backslash, double-quote, newline (mirrors event-ledger-cli.js's + * escapeLabelValue). */ +function escapeLabelValue(value) { + return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n"); +} + +/** buckets.perRepo is keyed by writeRateLimitRepoKey(actionClass, repoFullName) = "actionClass:repoFullName" + * (write-rate-limit.ts). actionClass is a fixed identifier (never contains ":"), so splitting on the FIRST + * colon recovers both parts even though repoFullName itself contains a "/". */ +function splitPerRepoKey(key) { + const separatorIndex = key.indexOf(":"); + if (separatorIndex === -1) return { actionClass: key, repoFullName: "" }; + return { actionClass: key.slice(0, separatorIndex), repoFullName: key.slice(separatorIndex + 1) }; +} + +// evaluateLocalRateLimit's own `remaining` field answers "how many MORE writes are allowed AFTER one more write +// right now" (rate-limit.ts: `remaining = allowed ? limit - effectiveCount - 1 : 0`) -- it is NOT current +// headroom. At count=2/limit=3 that field is already 0, identical to a fully exhausted count=3/limit=3 bucket, +// even though the count=2 bucket still has one write available. Recover true current headroom algebraically +// instead: when allowed, decision.remaining + 1 is exactly limit - effectiveCount (undo the "-1 for this next +// write" the decision already applied); when not allowed, headroom is 0. Every actionClass this loop reaches +// has already passed the DEFAULT_WRITE_RATE_LIMIT_POLICIES lookup above, so decision.limit is always one of the +// frozen, non-zero policy limits -- no zero-limit guard needed. +function remainingRatio(decision) { + const headroom = decision.allowed ? decision.remaining + 1 : 0; + return headroom / decision.limit; +} + +function collectRateLimitRows(buckets, nowMs) { + const rows = []; + for (const [actionClass, bucket] of Object.entries(buckets.global)) { + const config = DEFAULT_WRITE_RATE_LIMIT_POLICIES.global[actionClass]; + if (!config) continue; + rows.push({ + scope: "global", + actionClass, + repoFullName: "", + ratio: remainingRatio(evaluateLocalRateLimit(bucket, config, nowMs)), + }); + } + for (const [key, bucket] of Object.entries(buckets.perRepo)) { + const { actionClass, repoFullName } = splitPerRepoKey(key); + const config = DEFAULT_WRITE_RATE_LIMIT_POLICIES.perRepo[actionClass]; + if (!config) continue; + rows.push({ + scope: "per_repo", + actionClass, + repoFullName, + ratio: remainingRatio(evaluateLocalRateLimit(bucket, config, nowMs)), + }); + } + rows.sort((a, b) => { + if (a.scope !== b.scope) return a.scope.localeCompare(b.scope); + if (a.actionClass !== b.actionClass) return a.actionClass.localeCompare(b.actionClass); + return a.repoFullName.localeCompare(b.repoFullName); + }); + return rows; +} + +// DEFAULT_AMS_POLICY_SPEC.capLimits is a frozen, non-zero constant for every dimension -- no zero-limit guard +// needed, mirroring remainingRatio()'s reasoning above. +function collectCapUsageRows(capUsage) { + const report = evaluateGovernorCaps(capUsage, DEFAULT_AMS_POLICY_SPEC.capLimits); + return [ + { dimension: "budget", dimensionReport: report.budget }, + { dimension: "turns", dimensionReport: report.turns }, + { dimension: "elapsed_ms", dimensionReport: report.termination }, + ].map(({ dimension, dimensionReport }) => ({ + dimension, + ratio: dimensionReport.used / dimensionReport.limit, + })); +} + +/** + * @param {import("./governor-state.js").GovernorRateLimitState} rateLimitState + * @param {import("@loopover/engine").GovernorCapUsage} capUsage + * @param {number} nowMs + */ +export function renderGovernorMetrics(rateLimitState, capUsage, nowMs) { + const rateLimitRows = collectRateLimitRows(rateLimitState.buckets, nowMs); + const capRows = collectCapUsageRows(capUsage); + + const lines = [ + `# HELP ${GOVERNOR_RATE_LIMIT_REMAINING_RATIO} ${escapeMetricsHelpText( + "Remaining headroom in the governor's current write-rate-limit window, as a fraction of the configured limit (1 = empty bucket, 0 = exhausted). Evaluated against DEFAULT_WRITE_RATE_LIMIT_POLICIES.", + )}`, + `# TYPE ${GOVERNOR_RATE_LIMIT_REMAINING_RATIO} gauge`, + ]; + for (const row of rateLimitRows) { + const repoLabel = row.scope === "per_repo" ? `,repo="${escapeLabelValue(row.repoFullName)}"` : ""; + lines.push( + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="${row.scope}",action_class="${escapeLabelValue(row.actionClass)}"${repoLabel}} ${row.ratio}`, + ); + } + + lines.push( + `# HELP ${GOVERNOR_CAP_USAGE_RATIO} ${escapeMetricsHelpText( + "The governor's persisted cumulative cap usage as a fraction of DEFAULT_AMS_POLICY_SPEC.capLimits (1 = ceiling reached). dimension is one of budget|turns|elapsed_ms.", + )}`, + ); + lines.push(`# TYPE ${GOVERNOR_CAP_USAGE_RATIO} gauge`); + for (const row of capRows) { + lines.push(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="${row.dimension}"} ${row.ratio}`); + } + + return `${lines.join("\n")}\n`; +} + +async function withGovernorState(options, run) { + const ownsGovernorState = options.openGovernorState === undefined; + const governorState = (options.openGovernorState ?? openGovernorState)(); + try { + return run(governorState); + } finally { + if (ownsGovernorState) governorState.close(); + } +} + +export async function runGovernorMetrics(args, options = {}) { + if (args.length > 0) { + return reportCliFailure(argsWantJson(args), GOVERNOR_METRICS_USAGE); + } + + try { + return await withGovernorState(options, (governorState) => { + const nowMs = Number.isFinite(options.nowMs) ? options.nowMs : Date.now(); + const rateLimitState = governorState.loadRateLimitState(); + const capUsage = governorState.loadCapUsage(); + console.log(renderGovernorMetrics(rateLimitState, capUsage, nowMs).trimEnd()); + return 0; + }); + } catch (error) { + return reportCliFailure(argsWantJson(args), describeCliError(error)); + } +} diff --git a/packages/gittensory-miner/package.json b/packages/gittensory-miner/package.json index 65c1e05744..820afc0361 100644 --- a/packages/gittensory-miner/package.json +++ b/packages/gittensory-miner/package.json @@ -37,7 +37,7 @@ "expected-engine.version" ], "scripts": { - "build": "node --check bin/gittensory-miner.js && node --check bin/gittensory-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-run.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-conflict-resolver.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-open-pr.js && node --check lib/governor-pause-cli.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/logger.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/metrics-cli.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/policy-doc-cache.js && node --check lib/policy-verdict-cache.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-number-parse.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/purge-cli.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/stack-detection.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" + "build": "node --check bin/gittensory-miner.js && node --check bin/gittensory-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-run.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-conflict-resolver.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-metrics-cli.js && node --check lib/governor-open-pr.js && node --check lib/governor-pause-cli.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/logger.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/metrics-cli.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/policy-doc-cache.js && node --check lib/policy-verdict-cache.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-number-parse.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/purge-cli.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/stack-detection.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" }, "dependencies": { "@loopover/engine": "*", diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index 0a67a5bff1..eca4873db3 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -5,9 +5,10 @@ # # Every rule below is grounded ONLY in metrics the loopover app actually exports at # GET /metrics, plus the synthetic `up` metric Prometheus emits per scrape target. The -# one exception is the final `gittensory-miner-prediction` group (#5188), which targets -# the gittensory-miner's OWN separate scrape surface (see that group's own comment for why -# it still ships here and stays dormant until a miner scrape target exists). +# exceptions are the final three groups -- gittensory-miner-prediction (#5188), +# gittensory-miner-portfolio-queue (#5186), and gittensory-miner-governor (#5187) -- which +# target the gittensory-miner's OWN separate scrape surface (see each group's own comment +# for why it still ships here and stays dormant until a miner scrape target exists). # # Thresholds are sane defaults for a SMALL single-host self-host. Tune the numbers in # `expr` / `for` to your traffic — each is commented with what it means and how to adjust. @@ -702,3 +703,41 @@ groups: summary: "gittensory miner portfolio-queue backlog above 200" description: "{{ $value | printf \"%.0f\" }} portfolio-queue items are queued (sustained 30m) -- claiming/attempting is falling behind discovery." runbook: "Check whether `queue next` / `loop` is running at all (a laptop-mode miner only drains the queue while actively invoked), or whether WIP caps (#4850's --global-wip/--per-repo-wip) are set low enough to bottleneck throughput relative to discovery volume." + + # ── Governor rate-limit/budget threshold pressure (#5187) ───────────────── + # Same DORMANT-until-a-scrape-target-exists posture as the two miner groups above: an operator runs + # `gittensory-miner governor metrics` (packages/gittensory-miner/lib/governor-metrics-cli.js's + # renderGovernorMetrics) to a file/pushgateway their Prometheus scrapes. Absent gittensory_miner_governor_* + # series simply yield no result. Read-only: these rules alert on the governor's already-persisted rate-limit + # (#5134/write-rate-limit.ts) and cap-usage (#5134/budget-cap.ts) state; they do not gate, retry, or modify + # governor decision logic in any way (governor-chokepoint.js/governor-chokepoint-persisted.js are untouched). + - name: gittensory-miner-governor + rules: + - alert: GittensoryMinerGovernorRateLimitPressureHigh + # remaining_ratio is 1 for an empty bucket and 0 once a write-rate-limit bucket (global or per-repo, + # evaluated against DEFAULT_WRITE_RATE_LIMIT_POLICIES) is exhausted. Each bucket's own window is only + # 60s (the default policy's windowMs), so staying under 0.1 for a full 10m means the bucket has been + # refilled to >90% capacity across at least ten separate windows in a row -- sustained heavy write + # pressure, not one spike. + expr: gittensory_miner_governor_rate_limit_remaining_ratio < 0.1 + for: 10m + labels: + severity: warning + annotations: + summary: "gittensory miner governor write-rate-limit bucket is nearly exhausted" + description: "{{ $labels.scope }}/{{ $labels.action_class }}{{ with $labels.repo }} ({{ . }}){{ end }} has had under 10% of its write-rate-limit headroom remaining for 10m." + runbook: "Check `gittensory-miner governor metrics` for the affected scope/action_class and slow down that write class (fewer concurrent attempts, longer loop --cycle-delay-ms), or raise the relevant DEFAULT_WRITE_RATE_LIMIT_POLICIES ceiling if the current limit is genuinely too conservative for this install." + + - alert: GittensoryMinerGovernorCapUsageHigh + # cap_usage_ratio compares governor-state's persisted cumulative capUsage (budget spent / turns taken / + # elapsed session ms) against DEFAULT_AMS_POLICY_SPEC.capLimits, the same fleet-wide default loop-cli.js + # itself falls back to when a repo has no `.gittensory-ams.yml` override. 0.9 gives an operator a + # heads-up before evaluateGovernorCaps' own exceeded/kill_switch verdict actually halts the run. + expr: gittensory_miner_governor_cap_usage_ratio > 0.9 + for: 10m + labels: + severity: warning + annotations: + summary: "gittensory miner governor is approaching a run cap ceiling" + description: "The governor's {{ $labels.dimension }} cap usage has been over 90% of its configured limit for 10m." + runbook: "Run `gittensory-miner governor metrics` to see which dimension (budget/turns/elapsed_ms) is under pressure, and either let the current run finish (evaluateGovernorCaps will kill_switch/deny once the ceiling is actually reached) or raise the corresponding capLimits in `.gittensory-ams.yml` if the ceiling is too tight for this repo's normal attempts." diff --git a/test/unit/alerts-miner-governor-rate-limit-budget-pressure.test.ts b/test/unit/alerts-miner-governor-rate-limit-budget-pressure.test.ts new file mode 100644 index 0000000000..37de72195c --- /dev/null +++ b/test/unit/alerts-miner-governor-rate-limit-budget-pressure.test.ts @@ -0,0 +1,109 @@ +import { readFileSync } from "node:fs"; +import { parse as parseYaml } from "yaml"; +import { describe, expect, it } from "vitest"; +import { + GOVERNOR_CAP_USAGE_RATIO, + GOVERNOR_RATE_LIMIT_REMAINING_RATIO, +} from "../../packages/gittensory-miner/lib/governor-metrics-cli.js"; + +// Fixture for the GittensoryMinerGovernorRateLimitPressureHigh / GittensoryMinerGovernorCapUsageHigh alerts +// (#5187). This is the config-side equivalent of a `promtool test rules` harness (the repo ships no promtool +// dependency): it pins each rule's formula, threshold, and metric names to the real renderer surface so the +// alerts can't silently drift away from the metrics they consume. Mirrors +// alerts-miner-portfolio-queue-backlog.test.ts (#5186) and alerts-miner-prediction-calibration-drift.test.ts +// (#5188). +// +// Deliberately keyed off governor-metrics-cli.js's exported metric-name CONSTANTS (renderGovernorMetrics) +// rather than hardcoded strings: if that renderer ever renames a gauge, this test fails instead of the alert +// going quietly stale against a metric name that no longer exists. + +interface AlertRule { + alert: string; + expr: string; + for?: string; + labels?: { severity?: string }; + annotations?: { summary?: string; description?: string; runbook?: string }; +} +interface AlertGroup { + name: string; + rules: AlertRule[]; +} +interface AlertsDoc { + groups: AlertGroup[]; +} + +const alertsDoc = parseYaml(readFileSync("prometheus/rules/alerts.yml", "utf8")) as AlertsDoc; + +function findAlert(name: string): AlertRule { + for (const group of alertsDoc.groups) { + const rule = group.rules.find((r) => r.alert === name); + if (rule) return rule; + } + throw new Error(`alert ${name} not found in prometheus/rules/alerts.yml`); +} + +describe("GittensoryMinerGovernorRateLimitPressureHigh alert (#5187)", () => { + const rule = findAlert("GittensoryMinerGovernorRateLimitPressureHigh"); + const flat = rule.expr.replace(/\s+/g, " ").trim(); + + it("lives in its own miner-scoped rule group, separate from the loopover server groups", () => { + const group = alertsDoc.groups.find((g) => g.rules.some((r) => r.alert === rule.alert)); + expect(group?.name).toBe("gittensory-miner-governor"); + }); + + it("keys off the real renderer's rate-limit-remaining gauge, not an invented metric name", () => { + expect(GOVERNOR_RATE_LIMIT_REMAINING_RATIO).toBe("gittensory_miner_governor_rate_limit_remaining_ratio"); + expect(flat).toBe(`${GOVERNOR_RATE_LIMIT_REMAINING_RATIO} < 0.1`); + }); + + it("requires 10m of sustained pressure (>= 10 successive 60s windows) before firing", () => { + expect(rule.for).toBe("10m"); + }); + + it("never references any loopover_* server metric (a miner rule must not fire on server data)", () => { + expect(flat).not.toMatch(/loopover_/); + }); + + it("has warning severity and human-readable annotations", () => { + expect(rule.labels?.severity).toBe("warning"); + expect(rule.annotations?.summary).toBeTruthy(); + expect(rule.annotations?.description).toBeTruthy(); + expect(rule.annotations?.runbook).toBeTruthy(); + }); + + it("only reads governor state -- the runbook never instructs a governor decision-logic change", () => { + const runbook = rule.annotations?.runbook ?? ""; + expect(runbook).not.toMatch(/chokepoint/i); + expect(runbook).not.toMatch(/evaluateGovernorChokepointGate/); + }); +}); + +describe("GittensoryMinerGovernorCapUsageHigh alert (#5187)", () => { + const rule = findAlert("GittensoryMinerGovernorCapUsageHigh"); + const flat = rule.expr.replace(/\s+/g, " ").trim(); + + it("lives in the same miner-scoped governor rule group", () => { + const group = alertsDoc.groups.find((g) => g.rules.some((r) => r.alert === rule.alert)); + expect(group?.name).toBe("gittensory-miner-governor"); + }); + + it("keys off the real renderer's cap-usage gauge, thresholded at 90%", () => { + expect(GOVERNOR_CAP_USAGE_RATIO).toBe("gittensory_miner_governor_cap_usage_ratio"); + expect(flat).toBe(`${GOVERNOR_CAP_USAGE_RATIO} > 0.9`); + }); + + it("has a 10m sustain window and warning severity", () => { + expect(rule.for).toBe("10m"); + expect(rule.labels?.severity).toBe("warning"); + }); + + it("never references any loopover_* server metric (a miner rule must not fire on server data)", () => { + expect(flat).not.toMatch(/loopover_/); + }); + + it("only reads governor state -- the runbook never instructs a governor decision-logic change", () => { + const runbook = rule.annotations?.runbook ?? ""; + expect(runbook).not.toMatch(/chokepoint/i); + expect(runbook).not.toMatch(/evaluateGovernorChokepointGate/); + }); +}); diff --git a/test/unit/miner-governor-metrics-cli.test.ts b/test/unit/miner-governor-metrics-cli.test.ts new file mode 100644 index 0000000000..8833547f55 --- /dev/null +++ b/test/unit/miner-governor-metrics-cli.test.ts @@ -0,0 +1,229 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { closeDefaultGovernorState, openGovernorState } from "../../packages/gittensory-miner/lib/governor-state.js"; +import { + GOVERNOR_CAP_USAGE_RATIO, + GOVERNOR_RATE_LIMIT_REMAINING_RATIO, + renderGovernorMetrics, + runGovernorMetrics, +} from "../../packages/gittensory-miner/lib/governor-metrics-cli.js"; +import { runGovernorCli } from "../../packages/gittensory-miner/lib/governor-ledger-cli.js"; + +const roots: string[] = []; +const states: Array<{ close(): void }> = []; + +function tempGovernorState() { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-governor-metrics-cli-")); + roots.push(root); + const state = openGovernorState(join(root, "governor-state.sqlite3")); + states.push(state); + return state; +} + +afterEach(() => { + for (const state of states.splice(0)) state.close(); + closeDefaultGovernorState(); + vi.restoreAllMocks(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +const NOW = Date.parse("2026-07-13T12:00:00.000Z"); +const EMPTY_CAP_USAGE = { budgetSpent: 0, turnsTaken: 0, elapsedMs: 0 }; + +describe("renderGovernorMetrics() (#5187)", () => { + it("emits well-formed HELP/TYPE headers even with no bucket activity and zero cap usage", () => { + const output = renderGovernorMetrics({ buckets: { global: {}, perRepo: {} }, backoffAttempts: {} }, EMPTY_CAP_USAGE, NOW); + expect(output).toContain(`# HELP ${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}`); + expect(output).toContain(`# TYPE ${GOVERNOR_RATE_LIMIT_REMAINING_RATIO} gauge`); + expect(output).toContain(`# HELP ${GOVERNOR_CAP_USAGE_RATIO}`); + expect(output).toContain(`# TYPE ${GOVERNOR_CAP_USAGE_RATIO} gauge`); + expect(output).toContain(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="budget"} 0`); + expect(output).toContain(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="turns"} 0`); + expect(output).toContain(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="elapsed_ms"} 0`); + expect(output.endsWith("\n")).toBe(true); + }); + + it("renders a global bucket's TRUE current headroom, not evaluateLocalRateLimit's next-write-adjusted remaining", () => { + // global.open_pr policy: limit 30, windowMs 60_000. count=27 within the window -> effectiveCount=27, + // allowed=true (27<30). evaluateLocalRateLimit's own `remaining` field is limit-effectiveCount-1=2 (headroom + // AFTER one more hypothetical write), so the renderer must add 1 back to recover current headroom (3) before + // dividing by limit -- using `remaining` directly would under-report by exactly one slot. + const output = renderGovernorMetrics( + { buckets: { global: { open_pr: { count: 27, windowStartMs: NOW } }, perRepo: {} }, backoffAttempts: {} }, + EMPTY_CAP_USAGE, + NOW, + ); + expect(output).toContain( + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="global",action_class="open_pr"} ${3 / 30}`, + ); + }); + + it("renders a per-repo bucket's TRUE current headroom, splitting the composite actionClass:repo key", () => { + // perRepo.open_pr policy: limit 3, windowMs 60_000. count=2 within the window -> effectiveCount=2, + // allowed=true (2<3): ONE write is still allowed even though evaluateLocalRateLimit's own `remaining` field + // is already 0 at this count (3-2-1). Current headroom is 1, not 0 -- this is the exact case a prior + // version of this renderer got wrong (indistinguishable from a fully exhausted bucket). + const output = renderGovernorMetrics( + { + buckets: { global: {}, perRepo: { "open_pr:acme/widgets": { count: 2, windowStartMs: NOW } } }, + backoffAttempts: {}, + }, + EMPTY_CAP_USAGE, + NOW, + ); + expect(output).toContain( + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="per_repo",action_class="open_pr",repo="acme/widgets"} ${1 / 3}`, + ); + }); + + it("renders 0 headroom for a bucket that has genuinely reached its limit (count === limit, not allowed)", () => { + // perRepo.open_pr policy: limit 3. count=3 -> effectiveCount=3, allowed=false (3 is NOT < 3): this is the + // ACTUAL exhausted case, distinct from the count=2 "one write left" case above -- both must not render the + // same ratio, which is exactly the bug this test (and the one above) guards against together. + const output = renderGovernorMetrics( + { + buckets: { global: {}, perRepo: { "open_pr:acme/widgets": { count: 3, windowStartMs: NOW } } }, + backoffAttempts: {}, + }, + EMPTY_CAP_USAGE, + NOW, + ); + expect(output).toContain( + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="per_repo",action_class="open_pr",repo="acme/widgets"} 0`, + ); + }); + + it("skips a global/per-repo bucket whose actionClass has no DEFAULT_WRITE_RATE_LIMIT_POLICIES entry", () => { + const output = renderGovernorMetrics( + { + buckets: { + global: { unknown_action: { count: 5, windowStartMs: NOW } }, + perRepo: { "unknown_action:acme/widgets": { count: 1, windowStartMs: NOW } }, + }, + backoffAttempts: {}, + }, + EMPTY_CAP_USAGE, + NOW, + ); + expect(output).not.toContain("unknown_action"); + }); + + it("recovers actionClass/repo from a malformed per-repo key with no colon separator", () => { + const output = renderGovernorMetrics( + { buckets: { global: {}, perRepo: { malformed_no_colon: { count: 0, windowStartMs: NOW } } }, backoffAttempts: {} }, + EMPTY_CAP_USAGE, + NOW, + ); + // "malformed_no_colon" has no DEFAULT_WRITE_RATE_LIMIT_POLICIES.perRepo entry, so it is skipped -- this + // test only exists to exercise splitPerRepoKey's separatorIndex === -1 branch without throwing. + expect(output).not.toContain("malformed_no_colon"); + }); + + it("sorts series deterministically by scope, then action_class, then repo", () => { + const output = renderGovernorMetrics( + { + buckets: { + global: { comment: { count: 0, windowStartMs: NOW }, open_pr: { count: 0, windowStartMs: NOW } }, + perRepo: { + "open_pr:zeta/repo": { count: 0, windowStartMs: NOW }, + "open_pr:acme/repo": { count: 0, windowStartMs: NOW }, + }, + }, + backoffAttempts: {}, + }, + EMPTY_CAP_USAGE, + NOW, + ); + // count=0 for every bucket -> full current headroom (limit - 0 = limit), so the ratio is 1 for all of them. + const seriesLines = output.split("\n").filter((line) => line.startsWith(GOVERNOR_RATE_LIMIT_REMAINING_RATIO + "{")); + expect(seriesLines).toEqual([ + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="global",action_class="comment"} 1`, + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="global",action_class="open_pr"} 1`, + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="per_repo",action_class="open_pr",repo="acme/repo"} 1`, + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="per_repo",action_class="open_pr",repo="zeta/repo"} 1`, + ]); + }); + + it("renders cap-usage ratios against DEFAULT_AMS_POLICY_SPEC.capLimits (budget 5, turns 20, elapsedMs 1_800_000)", () => { + const output = renderGovernorMetrics( + { buckets: { global: {}, perRepo: {} }, backoffAttempts: {} }, + { budgetSpent: 4.5, turnsTaken: 20, elapsedMs: 900_000 }, + NOW, + ); + expect(output).toContain(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="budget"} 0.9`); + expect(output).toContain(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="turns"} 1`); + expect(output).toContain(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="elapsed_ms"} 0.5`); + }); +}); + +describe("runGovernorMetrics (#5187)", () => { + it("prints the rendered document from the real governor-state store", async () => { + const governorState = tempGovernorState(); + governorState.saveRateLimitState({ + buckets: { global: { open_pr: { count: 15, windowStartMs: NOW } }, perRepo: {} }, + backoffAttempts: {}, + }); + governorState.saveCapUsage({ budgetSpent: 1, turnsTaken: 2, elapsedMs: 3 }); + + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(await runGovernorMetrics([], { openGovernorState: () => governorState, nowMs: NOW })).toBe(0); + const output = String(log.mock.calls[0]?.[0]); + // limit 30, count 15 -> current headroom = 30 - 15 = 15, ratio = 15/30 = 0.5. + expect(output).toContain(`${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="global",action_class="open_pr"} 0.5`); + expect(output.endsWith("\n")).toBe(false); // console.log adds its own trailing newline + }); + + it("defaults nowMs to the real clock when not injected", async () => { + const governorState = tempGovernorState(); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(await runGovernorMetrics([], { openGovernorState: () => governorState })).toBe(0); + expect(String(log.mock.calls[0]?.[0])).toContain(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="budget"} 0`); + }); + + it("rejects unexpected positional args and surfaces a store failure", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + expect(await runGovernorMetrics(["extra"])).toBe(2); + expect(String(error.mock.calls[0]?.[0])).toContain("Usage: gittensory-miner governor metrics"); + + error.mockClear(); + expect( + await runGovernorMetrics([], { + openGovernorState: () => { + throw new Error("store_broken"); + }, + }), + ).toBe(2); + expect(error).toHaveBeenCalledWith("store_broken"); + }); + + it("reports a JSON-formatted usage error on stdout when --json is present alongside an extra arg", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(await runGovernorMetrics(["extra", "--json"])).toBe(2); + expect(JSON.parse(String(log.mock.calls[0]?.[0]))).toMatchObject({ ok: false }); + }); + + it("opens and closes the default on-disk governor state when no override is supplied", async () => { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-governor-metrics-cli-default-")); + roots.push(root); + const dbPath = join(root, "governor-state.sqlite3"); + const previousDbPath = process.env.GITTENSORY_MINER_GOVERNOR_STATE_DB; + process.env.GITTENSORY_MINER_GOVERNOR_STATE_DB = dbPath; + try { + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(await runGovernorMetrics([])).toBe(0); + expect(log).toHaveBeenCalled(); + } finally { + if (previousDbPath === undefined) delete process.env.GITTENSORY_MINER_GOVERNOR_STATE_DB; + else process.env.GITTENSORY_MINER_GOVERNOR_STATE_DB = previousDbPath; + } + }); + + it("runGovernorCli dispatches the metrics subcommand", async () => { + const governorState = tempGovernorState(); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(await runGovernorCli("metrics", [], { openGovernorState: () => governorState })).toBe(0); + expect(log).toHaveBeenCalled(); + }); +});