diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index a1655f6f32..8078bf49e6 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -4,7 +4,10 @@ # via the `rule_files: ["/etc/prometheus/rules/*.yml"]` glob in prometheus.yml. # # Every rule below is grounded ONLY in metrics the loopover app actually exports at -# GET /metrics, plus the synthetic `up` metric Prometheus emits per scrape target. +# GET /metrics, plus the synthetic `up` metric Prometheus emits per scrape target. The +# one exception is the final `gittensory-miner-prediction` group (#5188), which targets +# the gittensory-miner's OWN separate scrape surface (see that group's own comment for why +# it still ships here and stays dormant until a miner scrape target exists). # # Thresholds are sane defaults for a SMALL single-host self-host. Tune the numbers in # `expr` / `for` to your traffic — each is commented with what it means and how to adjust. @@ -615,3 +618,48 @@ groups: summary: "the Cloudflare D1 size/row-count probe is failing" description: "{{ $value | printf \"%.0f\" }} D1 Management API probe failure(s) over the last 1h (sustained 15m, part={{ $labels.part }}). The size/row-count gauges below may be stale." runbook: "Check CLOUDFLARE_D1_MONITOR_API_TOKEN is still valid and CLOUDFLARE_D1_MONITOR_ACCOUNT_ID/DATABASE_ID are correct. Tail logs for level=error event=d1_size_probe_error." + + # ── Miner prediction-calibration drift (#5188) ──────────────────────────── + # UNLIKE every group above, this rule targets the gittensory-MINER's own scrape surface, not the + # loopover server's GET /metrics. The miner is a local CLI (not a daemon), so an operator renders + # renderMinerPredictionMetrics (packages/gittensory-engine/src/miner-prediction-metrics.ts, #4264, wired + # into a command in #4838 over the calibration-report join built in #4849) to a file/pushgateway their + # Prometheus scrapes. The rule is therefore DORMANT until such a target exists: an absent + # gittensory_miner_prediction_* series simply yields no result (and the `> 0` denominator guard also makes + # the 0/0 case undefined), so it loads cleanly via `selfhost:validate-observability` and never false-fires + # on an install that has no miner scrape configured yet -- exactly like loopover-d1-storage stays silent + # without the D1 probe. + - name: gittensory-miner-prediction + rules: + - alert: GittensoryMinerPredictionCalibrationDrift + # Calibration = how often the miner's predicted gate conclusion matched the realized outcome. This is + # the miner counterpart of loopover-jobs' LoopoverHighJobFailureRatio and uses the identical + # bad/(bad+good) shape: the fraction of RESOLVED predictions that came back INCORRECT over the window. + # gittensory_miner_prediction_{correct,incorrect}_total only move for rows carrying a resolved outcome + # (renderMinerPredictionMetrics), so still-unresolved predictions never dilute the ratio. The trailing + # `> 0` on the denominator guards the 0/0 = NaN case (no resolved predictions in the window -> the rule + # stays inactive rather than firing on garbage), which is also what makes it degrade to silent when the + # metric is absent/unset. A wide 6h rate window smooths the miner's naturally sparse, bursty resolution + # cadence. + # + # TUNABLE THRESHOLD: 0.5 = alert once more than half of recently resolved predictions were wrong + # (calibration has drifted badly). This is the one knob to tune -- lower it (e.g. 0.35) for a stricter + # accuracy bar, raise it if the miner legitimately runs against harder repos. This follows the file's + # inline-documented-threshold convention (there are no recording rules here to hang a named constant on). + expr: | + ( + sum(rate(gittensory_miner_prediction_incorrect_total[6h])) + / + ( + sum(rate(gittensory_miner_prediction_correct_total[6h])) + + sum(rate(gittensory_miner_prediction_incorrect_total[6h])) + > 0 + ) + ) > 0.5 + for: 30m + labels: + severity: warning + annotations: + summary: "gittensory miner prediction calibration has drifted" + description: "{{ $value | humanizePercentage }} of the miner's recently resolved gate predictions were incorrect over the last 6h (sustained 30m). Predicted-gate accuracy has drifted out of tolerance." + runbook: "Compare gittensory_miner_prediction_correct_total vs gittensory_miner_prediction_incorrect_total and review the miner's own calibration report (`gittensory-miner calibration`). Sustained drift usually means the predicted-gate heuristics need recalibration against the repo(s) the miner now targets, or a recent upstream gate-behavior change the predictor hasn't caught up to." diff --git a/test/unit/alerts-miner-prediction-calibration-drift.test.ts b/test/unit/alerts-miner-prediction-calibration-drift.test.ts new file mode 100644 index 0000000000..6696a73217 --- /dev/null +++ b/test/unit/alerts-miner-prediction-calibration-drift.test.ts @@ -0,0 +1,82 @@ +import { readFileSync } from "node:fs"; +import { parse as parseYaml } from "yaml"; +import { describe, expect, it } from "vitest"; +import { + MINER_PREDICTION_CORRECT_TOTAL, + MINER_PREDICTION_INCORRECT_TOTAL, +} from "../../packages/gittensory-engine/src/miner-prediction-metrics"; + +// Fixture for the GittensoryMinerPredictionCalibrationDrift alert (#5188). This is the config-side +// equivalent of a `promtool test rules` harness (the repo ships no promtool dependency): it pins the +// rule's formula, threshold, and metric names to the real renderer surface so the alert can't silently +// drift away from the metrics it consumes. Mirrors alerts-job-failure-ratio-formula.test.ts (#3892). +// +// Deliberately keyed off the engine's exported metric-name CONSTANTS (renderMinerPredictionMetrics, +// packages/gittensory-engine/src/miner-prediction-metrics.ts) rather than hardcoded strings: if that +// renderer ever renames a counter, this test fails instead of the alert going quietly stale against a +// metric name that no longer exists. + +interface AlertRule { + alert: string; + expr: string; + for?: string; + labels?: { severity?: string }; + annotations?: { summary?: string; description?: string; runbook?: string }; +} +interface AlertGroup { + name: string; + rules: AlertRule[]; +} +interface AlertsDoc { + groups: AlertGroup[]; +} + +const alertsDoc = parseYaml(readFileSync("prometheus/rules/alerts.yml", "utf8")) as AlertsDoc; + +function findAlert(name: string): AlertRule { + for (const group of alertsDoc.groups) { + const rule = group.rules.find((r) => r.alert === name); + if (rule) return rule; + } + throw new Error(`alert ${name} not found in prometheus/rules/alerts.yml`); +} + +describe("GittensoryMinerPredictionCalibrationDrift alert (#5188)", () => { + const rule = findAlert("GittensoryMinerPredictionCalibrationDrift"); + const flat = rule.expr.replace(/\s+/g, " ").trim(); + + it("lives in its own miner-scoped rule group, separate from the loopover server groups", () => { + const group = alertsDoc.groups.find((g) => g.rules.some((r) => r.alert === rule.alert)); + expect(group?.name).toBe("gittensory-miner-prediction"); + }); + + it("keys off the miner renderer's real correct/incorrect counters, not invented metric names", () => { + expect(MINER_PREDICTION_CORRECT_TOTAL).toBe("gittensory_miner_prediction_correct_total"); + expect(MINER_PREDICTION_INCORRECT_TOTAL).toBe("gittensory_miner_prediction_incorrect_total"); + expect(flat).toContain(MINER_PREDICTION_CORRECT_TOTAL); + expect(flat).toContain(MINER_PREDICTION_INCORRECT_TOTAL); + }); + + it("uses the incorrect/(correct+incorrect) drift ratio shape over a 6h window", () => { + expect(flat).toMatch( + /sum\(rate\(gittensory_miner_prediction_incorrect_total\[6h\]\)\) \/ \( sum\(rate\(gittensory_miner_prediction_correct_total\[6h\]\)\) \+ sum\(rate\(gittensory_miner_prediction_incorrect_total\[6h\]\)\)/, + ); + }); + + it("guards the ratio against a 0/0 NaN so it degrades to silent when no predictions are resolved (invariant)", () => { + // The trailing `> 0` on the denominator is what makes an absent/empty calibration series yield no + // result instead of a false-positive alert. + expect(flat).toMatch(/> 0 \) \) > 0\.5/); + }); + + it("never references any loopover_* server metric (a miner rule must not fire on server data)", () => { + expect(flat).not.toMatch(/loopover_/); + }); + + it("has a sustain window, warning severity, and human-readable annotations", () => { + expect(rule.for).toBe("30m"); + expect(rule.labels?.severity).toBe("warning"); + expect(rule.annotations?.summary).toBeTruthy(); + expect(rule.annotations?.description).toBeTruthy(); + }); +});