diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index 3c4d6c261e..8e7ad3ac7d 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -39,22 +39,30 @@ groups: - name: gittensory-jobs rules: - alert: GittensoryHighJobFailureRatio - # Fraction of processed jobs that failed over the last 10m. The `> 0` guard on - # the denominator avoids a 0/0 = NaN (which never fires but is noise in /rules). - # 0.10 = 10% of jobs failing. Raise if your workload has expected transient - # failures; lower if any failure is a real problem. + # Fraction of jobs ATTEMPTED (failed + succeeded) that failed over the last 10m. + # gittensory_jobs_processed_total only increments on success, so the denominator + # must be failed+processed, not processed alone (#3892) -- dividing by processed + # alone computes failed:success and can read as high as 100% at a true 50% failure + # rate. The `> 0` guard on the ratio avoids a 0/0 = NaN (which never fires but is + # noise in /rules) when no jobs ran at all. 0.10 = 10% of jobs failing. Raise if + # your workload has expected transient failures; lower if any failure is a real + # problem. Mirrors the Grafana "Job Failure Rate" panel's formula. expr: | ( sum(rate(gittensory_jobs_failed_total[10m])) / - sum(rate(gittensory_jobs_processed_total[10m])) > 0 + ( + sum(rate(gittensory_jobs_failed_total[10m])) + + + sum(rate(gittensory_jobs_processed_total[10m])) + ) > 0 ) > 0.10 for: 15m labels: severity: warning annotations: summary: "gittensory job failure ratio above 10%" - description: "{{ $value | humanizePercentage }} of jobs processed in the last 10m failed (sustained 15m). Expected: well under 10%." + description: "{{ $value | humanizePercentage }} of jobs attempted in the last 10m failed (sustained 15m). Expected: well under 10%." runbook: "Tail logs for level=error job events (e.g. selfhost_cron_error). A spike usually means a bad upstream (GitHub API / AI provider / DB) or a poison payload — check what changed." - alert: GittensoryDeadLetterJobsGrowing diff --git a/test/unit/alerts-job-failure-ratio-formula.test.ts b/test/unit/alerts-job-failure-ratio-formula.test.ts new file mode 100644 index 0000000000..7b09b2c769 --- /dev/null +++ b/test/unit/alerts-job-failure-ratio-formula.test.ts @@ -0,0 +1,46 @@ +import { readFileSync } from "node:fs"; +import { parse as parseYaml } from "yaml"; +import { describe, expect, it } from "vitest"; + +// Regression test (#3892): GittensoryHighJobFailureRatio's expr used to divide failed-job rate by +// gittensory_jobs_processed_total alone. That metric only increments on SUCCESS (src/selfhost/pg-queue.ts, +// src/selfhost/sqlite-queue.ts), so the old expr computed failed:success, not a true failure percentage -- +// at a genuine 50% failure rate it evaluated to 100%. This pins the corrected failed/(failed+processed) +// shape (matching the Grafana "Job Failure Rate" panel's formula) so the bug can't silently return. + +interface AlertRule { + alert: string; + expr: string; +} +interface AlertGroup { + name: string; + rules: AlertRule[]; +} +interface AlertsDoc { + groups: AlertGroup[]; +} + +const alertsDoc = parseYaml(readFileSync("prometheus/rules/alerts.yml", "utf8")) as AlertsDoc; + +function findAlert(name: string): AlertRule { + for (const group of alertsDoc.groups) { + const rule = group.rules.find((r) => r.alert === name); + if (rule) return rule; + } + throw new Error(`alert ${name} not found in prometheus/rules/alerts.yml`); +} + +describe("GittensoryHighJobFailureRatio alert formula (#3892)", () => { + const expr = findAlert("GittensoryHighJobFailureRatio").expr; + const flat = expr.replace(/\s+/g, " ").trim(); + + it("divides failed by (failed + processed), not by processed alone", () => { + expect(flat).toMatch( + /sum\(rate\(gittensory_jobs_failed_total\[10m\]\)\) \/ \( sum\(rate\(gittensory_jobs_failed_total\[10m\]\)\) \+ sum\(rate\(gittensory_jobs_processed_total\[10m\]\)\) \) > 0/, + ); + }); + + it("still guards the ratio comparison against a 0/0 NaN before applying the 10% threshold", () => { + expect(flat).toMatch(/\) > 0 \) > 0\.10/); + }); +});