diff --git a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts index dcfb23097a..c41a9338b8 100644 --- a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts +++ b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts @@ -189,6 +189,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "GITHUB_INSTALLATION_CONCURRENCY_LIMIT", firstReference: "src/selfhost/installation-concurrency-admission.ts", }, + { + name: "GITTENSORY_ENABLE_PAGERDUTY", + firstReference: "src/services/notify-pagerduty.ts", + }, { name: "GITTENSORY_REPO_CONFIG_DIR", firstReference: "src/server.ts", @@ -341,6 +345,18 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "OTEL_TRACES_SAMPLER_ARG", firstReference: "src/selfhost/otel.ts", }, + { + name: "PAGERDUTY_COOLDOWN_MINUTES", + firstReference: "src/services/notify-pagerduty.ts", + }, + { + name: "PAGERDUTY_MIN_SEVERITY", + firstReference: "src/services/notify-pagerduty.ts", + }, + { + name: "PAGERDUTY_ROUTING_KEY", + firstReference: "src/services/notify-pagerduty.ts", + }, { name: "PGPOOL_MAX", firstReference: "src/selfhost/queue-common.ts", @@ -512,6 +528,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `GITHUB_INSTALLATION_CONCURRENCY_DEFER_MS` | `src/selfhost/installation-concurrency-admission.ts` |", "| `GITHUB_INSTALLATION_CONCURRENCY_ENABLED` | `src/selfhost/installation-concurrency-admission.ts` |", "| `GITHUB_INSTALLATION_CONCURRENCY_LIMIT` | `src/selfhost/installation-concurrency-admission.ts` |", + "| `GITTENSORY_ENABLE_PAGERDUTY` | `src/services/notify-pagerduty.ts` |", "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts` |", "| `GITTENSORY_VERSION` | `src/selfhost/otel.ts` |", "| `HOME` | `src/selfhost/ai.ts` |", @@ -550,6 +567,9 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `OTEL_TRACES_EXPORTER` | `src/selfhost/otel.ts` |", "| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts` |", "| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts` |", + "| `PAGERDUTY_COOLDOWN_MINUTES` | `src/services/notify-pagerduty.ts` |", + "| `PAGERDUTY_MIN_SEVERITY` | `src/services/notify-pagerduty.ts` |", + "| `PAGERDUTY_ROUTING_KEY` | `src/services/notify-pagerduty.ts` |", "| `PGPOOL_MAX` | `src/selfhost/queue-common.ts` |", "| `PGVECTOR_ENABLED` | `src/server.ts` |", "| `PORT` | `src/server.ts` |", diff --git a/scripts/gen-selfhost-env-reference.mjs b/scripts/gen-selfhost-env-reference.mjs index 7a3bc773d1..ab10fb6b62 100644 --- a/scripts/gen-selfhost-env-reference.mjs +++ b/scripts/gen-selfhost-env-reference.mjs @@ -9,6 +9,7 @@ export const DEFAULT_SOURCE_ROOTS = [ "src/selfhost", "src/server.ts", "src/services/notify-discord.ts", + "src/services/notify-pagerduty.ts", "scripts/build-selfhost.mjs", "scripts/migrate-selfhost-sqlite-to-postgres.ts", "scripts/smoke-observability-traces.mjs", diff --git a/src/env.d.ts b/src/env.d.ts index eb98c876cb..73fcf11298 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -218,6 +218,27 @@ declare global { /** Self-host Slack incoming-webhook URL (`https://hooks.slack.com/services/…`) — per-action notifications * (merged/closed/manual) for ANY repo. Sibling of DISCORD_WEBHOOK_URL; set either, both, or neither. */ SLACK_WEBHOOK_URL?: string; + /** Experimental (#4937/#5007): enables PagerDuty incident paging from src/services/notify-pagerduty.ts. + * Default OFF — unset/false keeps every export there a no-op. Truthy: `/^(1|true|yes|on)$/i`. */ + GITTENSORY_ENABLE_PAGERDUTY?: string; + /** Global fallback PagerDuty Events API v2 routing key (32 lowercase hex chars) for any repo not present in + * PAGERDUTY_REPO_ROUTING_KEYS (a JSON `{repoFullName: routingKey}` map, read directly off the env — same + * deliberately-untyped pattern as DISCORD_REPO_WEBHOOKS, since a free-form per-repo map isn't worth a + * formal interface field). Only read when GITTENSORY_ENABLE_PAGERDUTY is set. */ + PAGERDUTY_ROUTING_KEY?: string; + /** Alert-fatigue control: the minimum anomaly severity (`info` < `warning` < `error` < `critical`) that + * actually pages, for any repo not present in PAGERDUTY_REPO_MIN_SEVERITY (a JSON `{repoFullName: + * severity}` map, same deliberately-untyped pattern as PAGERDUTY_REPO_ROUTING_KEYS). Defaults to `error` + * when unset — the quietest safe default, so routine calibration nudges (gate/slop/recommendation drift) + * never page; only active-incident anomalies (review/failure bursts) do. Lower a specific repo's threshold + * via the map to page on its calibration nudges too. */ + PAGERDUTY_MIN_SEVERITY?: string; + /** Alert-fatigue control: minutes to suppress a REPEAT page for the same repo's ongoing anomaly condition + * (`dedup_key`) after one already paged, for any repo not present in PAGERDUTY_REPO_COOLDOWN_MINUTES (a + * JSON `{repoFullName: minutes}` map, same deliberately-untyped pattern as the other per-repo maps above). + * Defaults to 60 when unset. This is on top of PagerDuty's own `dedup_key` coalescing (which prevents + * duplicate *incidents*, not duplicate *pages* for a still-open one). */ + PAGERDUTY_COOLDOWN_MINUTES?: string; GITTENSORY_CONTRIBUTOR_ISSUE_TOKEN?: string; PRODUCT_USAGE_HASH_SALT?: string; GITTENSORY_API_TOKEN: string; diff --git a/src/review/ops-wire.ts b/src/review/ops-wire.ts index 0fe367341a..5416431afc 100644 --- a/src/review/ops-wire.ts +++ b/src/review/ops-wire.ts @@ -36,6 +36,7 @@ import { isAgentConfigured } from "../settings/autonomy"; import { resolveRepositorySettings } from "../settings/repository-settings"; import { loadGatePrecisionReport, type GatePrecisionReport } from "../services/gate-precision"; import { buildRepoOutcomeCalibration, type OutcomeCalibration } from "../services/outcome-calibration"; +import { triggerPagerDutyIncident, type PagerDutySeverity } from "../services/notify-pagerduty"; import { errorMessage, nowIso } from "../utils/json"; /** True when the ops observability surface is enabled. Flag-OFF (default) → every export below is a no-op / @@ -144,6 +145,29 @@ export function detectOutcomeAnomalies(snapshot: RepoOutcomeSnapshot): string[] return out; } +/** Classify one {@link detectOutcomeAnomalies} line by how urgently it needs a human, for PagerDuty's + * {@link resolvePagerDutyMinSeverity} gate. The three calibration-style anomalies (gate/slop/recommendation) + * are "worth recalibrating sometime" signals; the two burst anomalies are active-incident signals — the + * #ops-anomaly-metric Prometheus counter below already draws this same line. Matches on each anomaly's own + * fixed message prefix (see {@link detectOutcomeAnomalies}), so this never needs the detector's return type + * (`string[]`) to change and stays decoupled from its already-tested, OpenAPI-exposed shape. */ +export function classifyAnomalySeverity(line: string): PagerDutySeverity { + return line.startsWith("review burst:") || line.startsWith("review failure burst:") ? "error" : "warning"; +} + +/** The worst (highest-severity) anomaly in a non-empty list, for the PagerDuty summary + severity — so a + * repo with both a routine calibration nudge and an active-incident burst pages (if at all) at the burst's + * urgency, not whichever anomaly happened to sort first. */ +export function worstAnomaly(anomalies: string[]): { line: string; severity: PagerDutySeverity } { + const severityRank: Record = { info: 0, warning: 1, error: 2, critical: 3 }; + let best = { line: anomalies[0] ?? "ops anomaly detected", severity: classifyAnomalySeverity(anomalies[0] ?? "") }; + for (const line of anomalies) { + const severity = classifyAnomalySeverity(line); + if (severityRank[severity] > severityRank[best.severity]) best = { line, severity }; + } + return best; +} + // ── Cron alerts: scan gittensory's outcome data, emit a structured log on drift (flag-gated by the caller) ── /** The registered repos to scan. Scoped to REGISTERED repos (the ones gittensory actually tracks outcomes @@ -196,6 +220,27 @@ export async function runOpsAlerts(env: Env): Promise> // Structured log = gittensory's notify path (no Discord/operator webhook exists) AND the Sentry path // (level:"error" + an `event` field reaches forwardStructuredLogToSentry). One line per repo. console.error(JSON.stringify({ level: "error", event: "ops_anomaly", repo: repoFullName, at: nowIso(), anomalies })); + // Experimental PagerDuty paging (#4937): no-op unless GITTENSORY_ENABLE_PAGERDUTY is set AND a routing + // key resolves for this repo (resolvePagerDutyRoutingKey). ops_anomaly is this codebase's own existing + // "something needs a human" judgment call -- reusing it here (rather than paging on every + // captureError/captureReviewFailure call, which would need its own frequency/threshold policy first) + // keeps this narrow and low-risk. Pages at the WORST anomaly's severity; triggerPagerDutyIncident itself + // applies the min-severity floor (routine calibration nudges never page by default) and a cooldown (a + // still-ongoing anomaly across consecutive cron ticks does not re-page every tick) -- see its own + // comment for why alert fatigue needed both controls, not just PagerDuty's own dedup_key. Awaited (not + // fire-and-forget) so a page failure is captured within THIS tick's own error handling, not orphaned + // after runOpsAlerts has already returned -- triggerPagerDutyIncident itself never throws and bounds + // its own HTTP call to a 5s timeout, so this cannot hang the scan. This does not yet send a matching + // "resolve" event once anomalies clear (would need tracking previous-tick state) -- an operator + // currently resolves the incident manually once the underlying condition is fixed. + const worst = worstAnomaly(anomalies); + await triggerPagerDutyIncident(env, { + repoFullName, + summary: worst.line, + severity: worst.severity, + dedupKey: `ops_anomaly:${repoFullName}`, + customDetails: { anomalies }, + }); // #ops-anomaly-metric: Prometheus counterpart to the log line above so a self-host operator can alert on // /metrics instead of grepping Workers Logs. Scoped to reviewBurst/reviewFailureBurst -- the two anomalies // this module exists to catch fast (#orb-ci-stuck-repeat / #review-burst-blind-spot) -- rather than every diff --git a/src/services/notify-pagerduty.ts b/src/services/notify-pagerduty.ts new file mode 100644 index 0000000000..9dfec9c25f --- /dev/null +++ b/src/services/notify-pagerduty.ts @@ -0,0 +1,210 @@ +import { countRecentAuditEventsForActorAndTarget, recordAuditEvent } from "../db/repositories"; +import { errorMessage } from "../utils/json"; + +// PagerDuty Events API v2 (https://developer.pagerduty.com/docs/events-api-v2/overview/). Experimental, +// default-OFF (GITTENSORY_ENABLE_PAGERDUTY) — a self-host operator opts in per #4937's paging epic. +// Mirrors notify-discord.ts's per-repo routing precedence exactly: PAGERDUTY_REPO_ROUTING_KEYS (a JSON map, +// {repoFullName: routingKey}) takes priority over the single global PAGERDUTY_ROUTING_KEY fallback. Neither +// var is declared on the strict Env type (same asymmetry as DISCORD_REPO_WEBHOOKS) — a free-form per-repo +// JSON map isn't worth a formal interface field; the global fallbacks are, and are declared in env.d.ts. +// +// ALERT FATIGUE: paging is the loudest, most disruptive channel gittensory has — unlike a Discord post or a +// Sentry issue, it can wake someone up. Two independent controls keep it from crying wolf, on top of +// PagerDuty's own `dedup_key` coalescing (which prevents duplicate *incidents* but not duplicate *pages* for +// a still-open one): +// • MIN SEVERITY — a repo only pages once its worst detected condition meets PAGERDUTY_MIN_SEVERITY (global, +// default `error`) or its PAGERDUTY_REPO_MIN_SEVERITY override. Routine calibration nudges never page by +// default; only active-incident-grade anomalies do. +// • COOLDOWN — a repeat trigger for the SAME `dedup_key` within PAGERDUTY_COOLDOWN_MINUTES (global, default +// 60) or its PAGERDUTY_REPO_COOLDOWN_MINUTES override is suppressed, so a still-ongoing condition re-checked +// every cron tick doesn't re-page every tick. + +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; +// PagerDuty routing/integration keys are 32 lowercase hex characters. +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; +const DEFAULT_MIN_SEVERITY: PagerDutySeverity = "error"; +const DEFAULT_COOLDOWN_MINUTES = 60; + +/** True when the experimental PagerDuty integration is enabled. Flag-OFF (default) → every export below is a + * no-op. Truthy follows the codebase convention (`/^(1|true|yes|on)$/i`, same as isOpsEnabled/isSafetyEnabled). */ +export function isPagerDutyEnabled(env: { GITTENSORY_ENABLE_PAGERDUTY?: string | undefined }): boolean { + return /^(1|true|yes|on)$/i.test(env.GITTENSORY_ENABLE_PAGERDUTY ?? ""); +} + +function envString(env: Env, name: string): string | undefined { + const fromEnv = (env as unknown as Record)[name]; + if (typeof fromEnv === "string" && fromEnv.trim().length > 0) return fromEnv.trim(); + /* v8 ignore next 2 -- process.env is the self-host Node fallback; Worker/D1 tests pass values on Env. */ + const processEnv = (globalThis as unknown as { process?: { env?: Record } }).process?.env; + const fromProcess = processEnv?.[name]; + return typeof fromProcess === "string" && fromProcess.trim().length > 0 ? fromProcess.trim() : undefined; +} + +/** Parse a `{repoFullName: value}` JSON map off `envName`, lower-casing repo keys. Malformed/absent → `{}`. */ +function repoJsonMap(env: Env, envName: string): Record { + const raw = envString(env, envName); + if (!raw) return {}; + try { + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {}; + const out: Record = {}; + for (const [repo, value] of Object.entries(parsed)) { + out[repo.toLowerCase()] = value; + } + return out; + } catch { + return {}; + } +} + +export type PagerDutyRoutingResolution = + | { status: "configured"; routingKey: string; source: "repo_map" | "global" } + | { status: "disabled"; reason: "flag_off" | "missing_repo_key" | "invalid_repo_key" | "missing_global_key" | "invalid_global_key" }; + +/** Resolve the PagerDuty routing key for `repoFullName`: per-repo map entry, else the global fallback, else + * disabled. Mirrors {@link resolveDiscordWebhook}'s exact precedence and shape. */ +export function resolvePagerDutyRoutingKey(env: Env, repoFullName: string): PagerDutyRoutingResolution { + if (!isPagerDutyEnabled(env as unknown as { GITTENSORY_ENABLE_PAGERDUTY?: string | undefined })) { + return { status: "disabled", reason: "flag_off" }; + } + const repoKey = repoFullName.toLowerCase(); + const map = repoJsonMap(env, "PAGERDUTY_REPO_ROUTING_KEYS"); + if (Object.prototype.hasOwnProperty.call(map, repoKey)) { + const mapped = map[repoKey]; + const routingKey = typeof mapped === "string" ? mapped.trim() : ""; + return routingKey && ROUTING_KEY_RE.test(routingKey) + ? { status: "configured", routingKey, source: "repo_map" } + : { status: "disabled", reason: "invalid_repo_key" }; + } + const fallback = envString(env, "PAGERDUTY_ROUTING_KEY"); + return fallback && ROUTING_KEY_RE.test(fallback) + ? { status: "configured", routingKey: fallback, source: "global" } + : { status: "disabled", reason: fallback ? "invalid_global_key" : "missing_global_key" }; +} + +export type PagerDutySeverity = "critical" | "error" | "warning" | "info"; + +const SEVERITY_RANK: Record = { info: 0, warning: 1, error: 2, critical: 3 }; + +function isPagerDutySeverity(value: unknown): value is PagerDutySeverity { + return value === "critical" || value === "error" || value === "warning" || value === "info"; +} + +/** Resolve the minimum severity that pages for `repoFullName`: per-repo map entry, else the global override, + * else {@link DEFAULT_MIN_SEVERITY} — the quietest safe default, so an operator who never touches these vars + * still only gets paged for active-incident-grade conditions, never routine calibration nudges. */ +export function resolvePagerDutyMinSeverity(env: Env, repoFullName: string): PagerDutySeverity { + const map = repoJsonMap(env, "PAGERDUTY_REPO_MIN_SEVERITY"); + const mapped = map[repoFullName.toLowerCase()]; + if (isPagerDutySeverity(mapped)) return mapped; + const global = envString(env, "PAGERDUTY_MIN_SEVERITY"); + return isPagerDutySeverity(global) ? global : DEFAULT_MIN_SEVERITY; +} + +/** Coerce a JSON-map value or raw env string to a positive minute count; anything else (absent, zero, + * negative, non-numeric) is "not configured", not "zero cooldown". */ +function coercePositiveMinutes(value: unknown): number | null { + const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : null; +} + +/** Resolve the repeat-page cooldown (minutes) for `repoFullName`: per-repo map entry, else the global + * override, else {@link DEFAULT_COOLDOWN_MINUTES}. */ +export function resolvePagerDutyCooldownMinutes(env: Env, repoFullName: string): number { + const map = repoJsonMap(env, "PAGERDUTY_REPO_COOLDOWN_MINUTES"); + const repoMinutes = coercePositiveMinutes(map[repoFullName.toLowerCase()]); + if (repoMinutes != null) return repoMinutes; + const globalMinutes = coercePositiveMinutes(envString(env, "PAGERDUTY_COOLDOWN_MINUTES")); + return globalMinutes ?? DEFAULT_COOLDOWN_MINUTES; +} + +async function auditPagerDutyNotification( + env: Env, + params: { repoFullName: string; dedupKey: string }, + outcome: "completed" | "denied" | "error", + detail: string, + metadata: Record = {}, +): Promise { + await recordAuditEvent(env, { + eventType: "external_notification.pagerduty", + actor: "gittensory", + targetKey: params.dedupKey, + outcome, + detail, + metadata: { repoFullName: params.repoFullName, dedupKey: params.dedupKey, ...metadata }, + }).catch((error) => { + console.warn(JSON.stringify({ event: "pagerduty_notify_audit_failed", repo: params.repoFullName, message: errorMessage(error).slice(0, 120) })); + }); +} + +/** Trigger (or update, via PagerDuty's own `dedup_key` semantics — a repeat call with the SAME dedupKey + * updates the existing incident instead of opening a new one) a PagerDuty incident for `repoFullName`. + * Best-effort: never throws — a paging failure must never affect the caller's own work. No-op when the + * flag is off, no routing key resolves for this repo, `severity` doesn't meet the repo's configured + * {@link resolvePagerDutyMinSeverity} floor, or a page for this `dedupKey` already fired within the repo's + * {@link resolvePagerDutyCooldownMinutes} window. An explicitly-misconfigured key (present but invalid) and + * a below-threshold/cooldown-suppressed page are audited as `denied` so they're discoverable, while the + * common "not opted in" case stays silent (no audit-log noise for every repo that never configured + * PagerDuty). */ +export async function triggerPagerDutyIncident( + env: Env, + params: { + repoFullName: string; + summary: string; + severity: PagerDutySeverity; + dedupKey: string; + customDetails?: Record | undefined; + }, +): Promise { + const resolution = resolvePagerDutyRoutingKey(env, params.repoFullName); + if (resolution.status === "disabled") { + if (resolution.reason !== "flag_off") { + await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "denied", resolution.reason); + } + return; + } + + const minSeverity = resolvePagerDutyMinSeverity(env, params.repoFullName); + if (SEVERITY_RANK[params.severity] < SEVERITY_RANK[minSeverity]) { + await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "denied", "below_min_severity", { + severity: params.severity, + minSeverity, + }); + return; + } + + const cooldownMinutes = resolvePagerDutyCooldownMinutes(env, params.repoFullName); + const cooldownSinceIso = new Date(Date.now() - cooldownMinutes * 60 * 1000).toISOString(); + const recentPages = await countRecentAuditEventsForActorAndTarget(env, "gittensory", "external_notification.pagerduty", params.dedupKey, cooldownSinceIso); + if (recentPages > 0) { + await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "denied", "cooldown_active", { cooldownMinutes }); + return; + } + + try { + const response = await fetch(PAGERDUTY_EVENTS_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: resolution.routingKey, + event_action: "trigger", + dedup_key: params.dedupKey, + payload: { + summary: params.summary.slice(0, 1024), + source: "gittensory", + severity: params.severity, + timestamp: new Date().toISOString(), + component: params.repoFullName, + custom_details: params.customDetails, + }, + }), + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) throw new Error(`pagerduty_events_http_${response.status}`); + await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "completed", "triggered", { source: resolution.source }); + } catch (error) { + const message = errorMessage(error); + console.warn(JSON.stringify({ event: "pagerduty_trigger_failed", repo: params.repoFullName, message: message.slice(0, 200) })); + await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "error", message.slice(0, 280)); + } +} diff --git a/test/unit/notify-pagerduty.test.ts b/test/unit/notify-pagerduty.test.ts new file mode 100644 index 0000000000..0c03b5e5be --- /dev/null +++ b/test/unit/notify-pagerduty.test.ts @@ -0,0 +1,267 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + isPagerDutyEnabled, + resolvePagerDutyCooldownMinutes, + resolvePagerDutyMinSeverity, + resolvePagerDutyRoutingKey, + triggerPagerDutyIncident, +} from "../../src/services/notify-pagerduty"; +import { recordAuditEvent } from "../../src/db/repositories"; +import { createTestEnv } from "../helpers/d1"; + +const VALID_KEY = "a".repeat(32); +const REPO_KEY = "b".repeat(32); +const ORIG = { ...process.env }; + +afterEach(() => { + for (const key of Object.keys(process.env)) { + if (!(key in ORIG)) delete process.env[key]; + } + Object.assign(process.env, ORIG); + vi.unstubAllGlobals(); +}); + +function stubFetch(status = 202): Array<{ url: string; body: Record }> { + const calls: Array<{ url: string; body: Record }> = []; + vi.stubGlobal("fetch", async (url: RequestInfo | URL, init?: RequestInit) => { + calls.push({ url: String(url), body: init?.body ? (JSON.parse(String(init.body)) as Record) : {} }); + return new Response(null, { status }); + }); + return calls; +} + +const withEnv = (over: Record = {}): Env => Object.assign(createTestEnv(), over) as Env; +const enabledEnv = (over: Record = {}): Env => withEnv({ GITTENSORY_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: VALID_KEY, ...over }); + +async function pagerDutyAudit(env: Env): Promise> { + const rows = await env.DB.prepare("select outcome, detail, target_key, metadata_json from audit_events where event_type = ? order by created_at").bind("external_notification.pagerduty").all<{ + outcome: string; + detail: string; + target_key: string; + metadata_json: string; + }>(); + return rows.results ?? []; +} + +function trigger(env: Env, over: Partial<{ repoFullName: string; summary: string; severity: "critical" | "error" | "warning" | "info"; dedupKey: string; customDetails: Record }> = {}): Promise { + return triggerPagerDutyIncident(env, { + repoFullName: "acme/widgets", + summary: "ops anomaly detected", + severity: "error", + dedupKey: "ops_anomaly:acme/widgets", + ...over, + }); +} + +describe("isPagerDutyEnabled", () => { + it("accepts the codebase-standard truthy strings, case-insensitively", () => { + for (const value of ["1", "true", "YES", "On"]) expect(isPagerDutyEnabled({ GITTENSORY_ENABLE_PAGERDUTY: value })).toBe(true); + }); + it("treats anything else (including unset) as disabled", () => { + for (const value of [undefined, "", "0", "false", "nah"]) expect(isPagerDutyEnabled({ GITTENSORY_ENABLE_PAGERDUTY: value })).toBe(false); + }); +}); + +describe("resolvePagerDutyRoutingKey", () => { + it("flag off → disabled/flag_off, even with a valid global key set", () => { + expect(resolvePagerDutyRoutingKey(withEnv({ PAGERDUTY_ROUTING_KEY: VALID_KEY }), "acme/widgets")).toEqual({ status: "disabled", reason: "flag_off" }); + }); + + it("resolves a repo-specific PAGERDUTY_REPO_ROUTING_KEYS entry case-insensitively, over the global key", () => { + const env = enabledEnv({ PAGERDUTY_REPO_ROUTING_KEYS: JSON.stringify({ "acme/widgets": REPO_KEY }) }); + expect(resolvePagerDutyRoutingKey(env, "ACME/Widgets")).toEqual({ status: "configured", routingKey: REPO_KEY, source: "repo_map" }); + }); + + it("REGRESSION: an invalid repo-specific key suppresses instead of falling back to the global key", () => { + const env = enabledEnv({ PAGERDUTY_REPO_ROUTING_KEYS: JSON.stringify({ "acme/widgets": "not-hex" }) }); + expect(resolvePagerDutyRoutingKey(env, "acme/widgets")).toEqual({ status: "disabled", reason: "invalid_repo_key" }); + }); + + it("REGRESSION: a non-string or blank repo-map entry fails closed", () => { + const env = enabledEnv({ PAGERDUTY_REPO_ROUTING_KEYS: JSON.stringify({ "acme/widgets": 123, "acme/blank": " " }) }); + expect(resolvePagerDutyRoutingKey(env, "acme/widgets")).toEqual({ status: "disabled", reason: "invalid_repo_key" }); + expect(resolvePagerDutyRoutingKey(env, "acme/blank")).toEqual({ status: "disabled", reason: "invalid_repo_key" }); + }); + + it("ignores malformed or non-object PAGERDUTY_REPO_ROUTING_KEYS values and falls back to the global key", () => { + expect(resolvePagerDutyRoutingKey(enabledEnv({ PAGERDUTY_REPO_ROUTING_KEYS: "{not json" }), "acme/widgets")).toEqual({ status: "configured", routingKey: VALID_KEY, source: "global" }); + expect(resolvePagerDutyRoutingKey(enabledEnv({ PAGERDUTY_REPO_ROUTING_KEYS: "null" }), "acme/widgets")).toEqual({ status: "configured", routingKey: VALID_KEY, source: "global" }); + expect(resolvePagerDutyRoutingKey(enabledEnv({ PAGERDUTY_REPO_ROUTING_KEYS: "123" }), "acme/widgets")).toEqual({ status: "configured", routingKey: VALID_KEY, source: "global" }); + expect(resolvePagerDutyRoutingKey(enabledEnv({ PAGERDUTY_REPO_ROUTING_KEYS: "[]" }), "acme/widgets")).toEqual({ status: "configured", routingKey: VALID_KEY, source: "global" }); + }); + + it("unmapped repo + no global key → disabled/missing_global_key", () => { + expect(resolvePagerDutyRoutingKey(withEnv({ GITTENSORY_ENABLE_PAGERDUTY: "1" }), "acme/widgets")).toEqual({ status: "disabled", reason: "missing_global_key" }); + }); + + it("unmapped repo + invalid global key → disabled/invalid_global_key", () => { + expect(resolvePagerDutyRoutingKey(enabledEnv({ PAGERDUTY_ROUTING_KEY: "not-hex" }), "acme/widgets")).toEqual({ status: "disabled", reason: "invalid_global_key" }); + }); + + it("uses process.env as a self-host fallback for the routing key when the runtime Env object does not carry it", () => { + process.env.PAGERDUTY_ROUTING_KEY = VALID_KEY; + expect(resolvePagerDutyRoutingKey(withEnv({ GITTENSORY_ENABLE_PAGERDUTY: "1" }), "acme/widgets")).toEqual({ status: "configured", routingKey: VALID_KEY, source: "global" }); + }); +}); + +describe("resolvePagerDutyMinSeverity", () => { + it("a valid repo-map entry wins over the global default", () => { + const env = withEnv({ PAGERDUTY_REPO_MIN_SEVERITY: JSON.stringify({ "acme/widgets": "warning" }), PAGERDUTY_MIN_SEVERITY: "critical" }); + expect(resolvePagerDutyMinSeverity(env, "acme/widgets")).toBe("warning"); + }); + + it("an invalid/absent repo entry falls back to a valid global override", () => { + expect(resolvePagerDutyMinSeverity(withEnv({ PAGERDUTY_MIN_SEVERITY: "info" }), "acme/widgets")).toBe("info"); + const env = withEnv({ PAGERDUTY_REPO_MIN_SEVERITY: JSON.stringify({ "acme/widgets": "not-a-severity" }), PAGERDUTY_MIN_SEVERITY: "critical" }); + expect(resolvePagerDutyMinSeverity(env, "acme/widgets")).toBe("critical"); + }); + + it("no repo entry + no/invalid global → defaults to error (the quietest safe default)", () => { + expect(resolvePagerDutyMinSeverity(withEnv(), "acme/widgets")).toBe("error"); + expect(resolvePagerDutyMinSeverity(withEnv({ PAGERDUTY_MIN_SEVERITY: "not-a-severity" }), "acme/widgets")).toBe("error"); + }); +}); + +describe("resolvePagerDutyCooldownMinutes", () => { + it("a valid repo-map entry (number or numeric string) wins over the global default", () => { + expect(resolvePagerDutyCooldownMinutes(withEnv({ PAGERDUTY_REPO_COOLDOWN_MINUTES: JSON.stringify({ "acme/widgets": 15 }), PAGERDUTY_COOLDOWN_MINUTES: "120" }), "acme/widgets")).toBe(15); + expect(resolvePagerDutyCooldownMinutes(withEnv({ PAGERDUTY_REPO_COOLDOWN_MINUTES: JSON.stringify({ "acme/widgets": "30" }) }), "acme/widgets")).toBe(30); + }); + + it("a zero/negative/non-numeric repo entry falls back to a valid global override", () => { + expect(resolvePagerDutyCooldownMinutes(withEnv({ PAGERDUTY_COOLDOWN_MINUTES: "45" }), "acme/widgets")).toBe(45); + for (const bad of [0, -5, "nope"]) { + const env = withEnv({ PAGERDUTY_REPO_COOLDOWN_MINUTES: JSON.stringify({ "acme/widgets": bad }), PAGERDUTY_COOLDOWN_MINUTES: "45" }); + expect(resolvePagerDutyCooldownMinutes(env, "acme/widgets")).toBe(45); + } + }); + + it("no repo entry + no/invalid global → defaults to 60 minutes", () => { + expect(resolvePagerDutyCooldownMinutes(withEnv(), "acme/widgets")).toBe(60); + expect(resolvePagerDutyCooldownMinutes(withEnv({ PAGERDUTY_COOLDOWN_MINUTES: "not-a-number" }), "acme/widgets")).toBe(60); + }); +}); + +describe("triggerPagerDutyIncident — flag/routing gate", () => { + it("flag off → no fetch and no audit row (silent, no log noise for repos that never opted in)", async () => { + const calls = stubFetch(); + const env = withEnv(); + await trigger(env); + expect(calls).toEqual([]); + expect(await pagerDutyAudit(env)).toEqual([]); + }); + + it("flag on, no routing key resolves → no fetch, audited denied/missing_global_key", async () => { + const calls = stubFetch(); + const env = withEnv({ GITTENSORY_ENABLE_PAGERDUTY: "1" }); + await trigger(env); + expect(calls).toEqual([]); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "denied", detail: "missing_global_key" })]); + }); + + it("audit failures are best-effort and never throw", async () => { + const calls = stubFetch(); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect(trigger({ GITTENSORY_ENABLE_PAGERDUTY: "1" } as Env)).resolves.toBeUndefined(); + expect(calls).toEqual([]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("pagerduty_notify_audit_failed")); + warn.mockRestore(); + }); +}); + +describe("triggerPagerDutyIncident — min-severity gate (alert fatigue control #1)", () => { + it("a severity below the default min (error) never pages", async () => { + const calls = stubFetch(); + const env = enabledEnv(); + await trigger(env, { severity: "warning" }); + expect(calls).toEqual([]); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "denied", detail: "below_min_severity" })]); + }); + + it("a severity meeting the default min (error) pages", async () => { + const calls = stubFetch(); + await trigger(enabledEnv(), { severity: "error" }); + expect(calls).toHaveLength(1); + }); + + it("a per-repo override lowers the floor so a warning-severity anomaly pages", async () => { + const calls = stubFetch(); + const env = enabledEnv({ PAGERDUTY_REPO_MIN_SEVERITY: JSON.stringify({ "acme/widgets": "warning" }) }); + await trigger(env, { severity: "warning" }); + expect(calls).toHaveLength(1); + }); +}); + +describe("triggerPagerDutyIncident — cooldown gate (alert fatigue control #2)", () => { + it("a repeat trigger for the same dedupKey within the cooldown window is suppressed", async () => { + const calls = stubFetch(); + const env = enabledEnv(); + await trigger(env); + await trigger(env); + expect(calls).toHaveLength(1); + const rows = await pagerDutyAudit(env); + expect(rows.map((r) => r.outcome)).toEqual(["completed", "denied"]); + expect(rows[1]).toEqual(expect.objectContaining({ detail: "cooldown_active" })); + }); + + it("a page for a DIFFERENT dedupKey on the same repo is not suppressed by the first one's cooldown", async () => { + const calls = stubFetch(); + const env = enabledEnv(); + await trigger(env, { dedupKey: "ops_anomaly:acme/widgets" }); + await trigger(env, { dedupKey: "some_other_condition:acme/widgets" }); + expect(calls).toHaveLength(2); + }); + + it("a page whose only prior trigger is OLDER than the cooldown window is not suppressed", async () => { + const calls = stubFetch(); + const env = enabledEnv(); + const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString(); + await recordAuditEvent(env, { + eventType: "external_notification.pagerduty", + actor: "gittensory", + targetKey: "ops_anomaly:acme/widgets", + outcome: "completed", + detail: "triggered", + metadata: {}, + createdAt: twoHoursAgo, + }); + await trigger(env); // default cooldown is 60 minutes; the seeded row is 2 hours old + expect(calls).toHaveLength(1); + }); +}); + +describe("triggerPagerDutyIncident — HTTP delivery", () => { + it("posts the PagerDuty Events API v2 payload and audits completed on success", async () => { + const calls = stubFetch(202); + const env = enabledEnv(); + await trigger(env, { severity: "critical", summary: "review burst on acme/widgets", customDetails: { anomalies: ["a", "b"] } }); + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe("https://events.pagerduty.com/v2/enqueue"); + expect(calls[0]?.body).toMatchObject({ + routing_key: VALID_KEY, + event_action: "trigger", + dedup_key: "ops_anomaly:acme/widgets", + payload: { summary: "review burst on acme/widgets", source: "gittensory", severity: "critical", component: "acme/widgets", custom_details: { anomalies: ["a", "b"] } }, + }); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "completed", detail: "triggered" })]); + }); + + it("a non-ok response is audited as an error and never throws", async () => { + stubFetch(500); + const env = enabledEnv(); + await expect(trigger(env)).resolves.toBeUndefined(); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "error" })]); + }); + + it("a network failure is audited as an error and never throws", async () => { + vi.stubGlobal("fetch", async () => { + throw new Error("network down"); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const env = enabledEnv(); + await expect(trigger(env)).resolves.toBeUndefined(); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "error", detail: expect.stringContaining("network down") })]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("pagerduty_trigger_failed")); + warn.mockRestore(); + }); +}); diff --git a/test/unit/ops-wire.test.ts b/test/unit/ops-wire.test.ts index 36952aa473..086d468302 100644 --- a/test/unit/ops-wire.test.ts +++ b/test/unit/ops-wire.test.ts @@ -2,11 +2,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createApp } from "../../src/api/routes"; import { recordAiUsageEvent, recordAuditEvent, recordGateBlockOutcome, upsertPullRequestFromGitHub } from "../../src/db/repositories"; import { + classifyAnomalySeverity, computeOpsStats, detectOutcomeAnomalies, isOpsEnabled, type RepoOutcomeSnapshot, runOpsAlerts, + worstAnomaly, } from "../../src/review/ops-wire"; import { counterValue, resetMetrics, setSelfHostedMetricsMode } from "../../src/selfhost/metrics"; import { createTestEnv } from "../helpers/d1"; @@ -162,6 +164,39 @@ describe("detectOutcomeAnomalies — over gittensory's own outcome data", () => }); }); +describe("classifyAnomalySeverity — PagerDuty min-severity classification", () => { + it("classifies the two burst anomalies as error (active-incident grade)", () => { + expect(classifyAnomalySeverity("review burst: owner/repo#42 published 9 review surfaces in the last 2h")).toBe("error"); + expect(classifyAnomalySeverity("review failure burst: owner/repo#42 produced 4 inconclusive calls")).toBe("error"); + }); + + it("classifies the three calibration-style anomalies as warning (worth recalibrating sometime)", () => { + expect(classifyAnomalySeverity("gate false-positive spike: `slop_risk` blocked 10 PR(s)")).toBe("warning"); + expect(classifyAnomalySeverity("slop score NOT discriminating (30 resolved PRs)")).toBe("warning"); + expect(classifyAnomalySeverity("recommendations not panning out: 8/10 resolved outcomes were negative")).toBe("warning"); + }); +}); + +describe("worstAnomaly — highest-severity anomaly wins for the PagerDuty page", () => { + it("a single anomaly is its own worst", () => { + expect(worstAnomaly(["slop score NOT discriminating (30 resolved PRs)"])).toEqual({ line: "slop score NOT discriminating (30 resolved PRs)", severity: "warning" }); + }); + + it("a later, higher-severity anomaly overtakes an earlier lower-severity one", () => { + const anomalies = ["gate false-positive spike: `slop_risk` blocked 10 PR(s)", "review burst: owner/repo#42 published 9 review surfaces in the last 2h"]; + expect(worstAnomaly(anomalies)).toEqual({ line: anomalies[1], severity: "error" }); + }); + + it("a later, lower-or-equal-severity anomaly never demotes the current worst", () => { + const anomalies = ["review burst: owner/repo#42 published 9 review surfaces in the last 2h", "slop score NOT discriminating (30 resolved PRs)"]; + expect(worstAnomaly(anomalies)).toEqual({ line: anomalies[0], severity: "error" }); + }); + + it("an empty list falls back to a generic line (defensive — runOpsAlerts never calls this with one)", () => { + expect(worstAnomaly([])).toEqual({ line: "ops anomaly detected", severity: "warning" }); + }); +}); + // ── DB-backed cron + endpoint integration over the real migrated schema ───────────────────────────────────── // Mark a repo registered so opsScanRepos picks it up (the registry sets is_registered=1; we seed it directly). @@ -357,6 +392,64 @@ describe("runOpsAlerts — cron path over gittensory's outcome data", () => { expect(found).toEqual({}); expect(errors.mock.calls.map((c) => String(c[0])).some((line) => line.includes("ops_anomaly_error"))).toBe(true); }); + + // ── Experimental PagerDuty paging (#4937): fatigue-controlled wiring on top of the anomaly scan ────────── + const PD_KEY = "a".repeat(32); + function stubPagerDutyFetch(status = 202): Array<{ body: { dedup_key: string; payload: { summary: string; severity: string } } }> { + const calls: Array<{ body: { dedup_key: string; payload: { summary: string; severity: string } } }> = []; + vi.stubGlobal("fetch", async (_url: RequestInfo | URL, init?: RequestInit) => { + calls.push({ body: JSON.parse(String(init?.body)) }); + return new Response(null, { status }); + }); + return calls; + } + + it("pages at the WORST anomaly's severity, not whichever one happened to sort first", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv({ GITTENSORY_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: PD_KEY }); + await seedRegisteredRepo(env, "owner/repo"); + // A calibration nudge (warning-grade) AND a review burst (error-grade) on the same repo, same tick. + await seedGateFalsePositiveAnomaly(env, "owner/repo"); + for (let i = 0; i < 7; i += 1) { + await recordAuditEvent(env, { eventType: "github_app.pr_public_surface_published", actor: "contributor", targetKey: "owner/repo#99", outcome: "completed" }); + } + vi.spyOn(console, "error").mockImplementation(() => {}); + + await runOpsAlerts(env); + + expect(calls).toHaveLength(1); + expect(calls[0]?.body.payload.severity).toBe("error"); + expect(calls[0]?.body.payload.summary).toMatch(/review burst/); + expect(calls[0]?.body.dedup_key).toBe("ops_anomaly:owner/repo"); + }); + + it("does NOT page for a repo whose only anomaly is a routine calibration nudge (default min-severity floor)", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv({ GITTENSORY_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: PD_KEY }); + await seedRegisteredRepo(env, "owner/repo"); + await seedGateFalsePositiveAnomaly(env, "owner/repo"); // warning-grade only, no burst + vi.spyOn(console, "error").mockImplementation(() => {}); + + const found = await runOpsAlerts(env); + + expect(found["owner/repo"]?.some((a) => /gate false-positive spike/.test(a))).toBe(true); // still logged/Sentry-visible + expect(calls).toEqual([]); // but never paged — below the default error floor + }); + + it("does NOT page at all when GITTENSORY_ENABLE_PAGERDUTY is unset (default OFF, byte-identical to today)", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv(); // no PagerDuty env vars + await seedRegisteredRepo(env, "owner/repo"); + for (let i = 0; i < 7; i += 1) { + await recordAuditEvent(env, { eventType: "github_app.pr_public_surface_published", actor: "contributor", targetKey: "owner/repo#99", outcome: "completed" }); + } + vi.spyOn(console, "error").mockImplementation(() => {}); + + const found = await runOpsAlerts(env); + + expect(found["owner/repo"]?.some((a) => /review burst/.test(a))).toBe(true); + expect(calls).toEqual([]); + }); }); describe("computeOpsStats — cross-repo outcome aggregate", () => {