feat(ops): add fatigue-controlled PagerDuty paging for ops anomalies

apps/gittensory-ui/src/lib/selfhost-env-reference.ts

@@ -189,6 +189,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "GITHUB_INSTALLATION_CONCURRENCY_LIMIT",
     firstReference: "src/selfhost/installation-concurrency-admission.ts",
   },
+  {
+    name: "GITTENSORY_ENABLE_PAGERDUTY",
+    firstReference: "src/services/notify-pagerduty.ts",
+  },
   {
     name: "GITTENSORY_REPO_CONFIG_DIR",
     firstReference: "src/server.ts",
@@ -341,6 +345,18 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "OTEL_TRACES_SAMPLER_ARG",
     firstReference: "src/selfhost/otel.ts",
   },
+  {
+    name: "PAGERDUTY_COOLDOWN_MINUTES",
+    firstReference: "src/services/notify-pagerduty.ts",
+  },
+  {
+    name: "PAGERDUTY_MIN_SEVERITY",
+    firstReference: "src/services/notify-pagerduty.ts",
+  },
+  {
+    name: "PAGERDUTY_ROUTING_KEY",
+    firstReference: "src/services/notify-pagerduty.ts",
+  },
   {
     name: "PGPOOL_MAX",
     firstReference: "src/selfhost/queue-common.ts",
@@ -512,6 +528,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `GITHUB_INSTALLATION_CONCURRENCY_DEFER_MS` | `src/selfhost/installation-concurrency-admission.ts` |",
   "| `GITHUB_INSTALLATION_CONCURRENCY_ENABLED` | `src/selfhost/installation-concurrency-admission.ts` |",
   "| `GITHUB_INSTALLATION_CONCURRENCY_LIMIT` | `src/selfhost/installation-concurrency-admission.ts` |",
+  "| `GITTENSORY_ENABLE_PAGERDUTY` | `src/services/notify-pagerduty.ts` |",
   "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts` |",
   "| `GITTENSORY_VERSION` | `src/selfhost/otel.ts` |",
   "| `HOME` | `src/selfhost/ai.ts` |",
@@ -550,6 +567,9 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `OTEL_TRACES_EXPORTER` | `src/selfhost/otel.ts` |",
   "| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts` |",
   "| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts` |",
+  "| `PAGERDUTY_COOLDOWN_MINUTES` | `src/services/notify-pagerduty.ts` |",
+  "| `PAGERDUTY_MIN_SEVERITY` | `src/services/notify-pagerduty.ts` |",
+  "| `PAGERDUTY_ROUTING_KEY` | `src/services/notify-pagerduty.ts` |",
   "| `PGPOOL_MAX` | `src/selfhost/queue-common.ts` |",
   "| `PGVECTOR_ENABLED` | `src/server.ts` |",
   "| `PORT` | `src/server.ts` |",

scripts/gen-selfhost-env-reference.mjs

@@ -9,6 +9,7 @@ export const DEFAULT_SOURCE_ROOTS = [
   "src/selfhost",
   "src/server.ts",
   "src/services/notify-discord.ts",
+  "src/services/notify-pagerduty.ts",
   "scripts/build-selfhost.mjs",
   "scripts/migrate-selfhost-sqlite-to-postgres.ts",
   "scripts/smoke-observability-traces.mjs",

src/env.d.ts

@@ -218,6 +218,27 @@ declare global {
     /** Self-host Slack incoming-webhook URL (`https://hooks.slack.com/services/…`) — per-action notifications
      *  (merged/closed/manual) for ANY repo. Sibling of DISCORD_WEBHOOK_URL; set either, both, or neither. */
     SLACK_WEBHOOK_URL?: string;
+    /** Experimental (#4937/#5007): enables PagerDuty incident paging from src/services/notify-pagerduty.ts.
+     *  Default OFF — unset/false keeps every export there a no-op. Truthy: `/^(1|true|yes|on)$/i`. */
+    GITTENSORY_ENABLE_PAGERDUTY?: string;
+    /** Global fallback PagerDuty Events API v2 routing key (32 lowercase hex chars) for any repo not present in
+     *  PAGERDUTY_REPO_ROUTING_KEYS (a JSON `{repoFullName: routingKey}` map, read directly off the env — same
+     *  deliberately-untyped pattern as DISCORD_REPO_WEBHOOKS, since a free-form per-repo map isn't worth a
+     *  formal interface field). Only read when GITTENSORY_ENABLE_PAGERDUTY is set. */
+    PAGERDUTY_ROUTING_KEY?: string;
+    /** Alert-fatigue control: the minimum anomaly severity (`info` < `warning` < `error` < `critical`) that
+     *  actually pages, for any repo not present in PAGERDUTY_REPO_MIN_SEVERITY (a JSON `{repoFullName:
+     *  severity}` map, same deliberately-untyped pattern as PAGERDUTY_REPO_ROUTING_KEYS). Defaults to `error`
+     *  when unset — the quietest safe default, so routine calibration nudges (gate/slop/recommendation drift)
+     *  never page; only active-incident anomalies (review/failure bursts) do. Lower a specific repo's threshold
+     *  via the map to page on its calibration nudges too. */
+    PAGERDUTY_MIN_SEVERITY?: string;
+    /** Alert-fatigue control: minutes to suppress a REPEAT page for the same repo's ongoing anomaly condition
+     *  (`dedup_key`) after one already paged, for any repo not present in PAGERDUTY_REPO_COOLDOWN_MINUTES (a
+     *  JSON `{repoFullName: minutes}` map, same deliberately-untyped pattern as the other per-repo maps above).
+     *  Defaults to 60 when unset. This is on top of PagerDuty's own `dedup_key` coalescing (which prevents
+     *  duplicate *incidents*, not duplicate *pages* for a still-open one). */
+    PAGERDUTY_COOLDOWN_MINUTES?: string;
     GITTENSORY_CONTRIBUTOR_ISSUE_TOKEN?: string;
     PRODUCT_USAGE_HASH_SALT?: string;
     GITTENSORY_API_TOKEN: string;

src/review/ops-wire.ts

@@ -36,6 +36,7 @@ import { isAgentConfigured } from "../settings/autonomy";
 import { resolveRepositorySettings } from "../settings/repository-settings";
 import { loadGatePrecisionReport, type GatePrecisionReport } from "../services/gate-precision";
 import { buildRepoOutcomeCalibration, type OutcomeCalibration } from "../services/outcome-calibration";
+import { triggerPagerDutyIncident, type PagerDutySeverity } from "../services/notify-pagerduty";
 import { errorMessage, nowIso } from "../utils/json";
 
 /** True when the ops observability surface is enabled. Flag-OFF (default) → every export below is a no-op /
@@ -144,6 +145,29 @@ export function detectOutcomeAnomalies(snapshot: RepoOutcomeSnapshot): string[]
   return out;
 }
 
+/** Classify one {@link detectOutcomeAnomalies} line by how urgently it needs a human, for PagerDuty's
+ *  {@link resolvePagerDutyMinSeverity} gate. The three calibration-style anomalies (gate/slop/recommendation)
+ *  are "worth recalibrating sometime" signals; the two burst anomalies are active-incident signals — the
+ *  #ops-anomaly-metric Prometheus counter below already draws this same line. Matches on each anomaly's own
+ *  fixed message prefix (see {@link detectOutcomeAnomalies}), so this never needs the detector's return type
+ *  (`string[]`) to change and stays decoupled from its already-tested, OpenAPI-exposed shape. */
+export function classifyAnomalySeverity(line: string): PagerDutySeverity {
+  return line.startsWith("review burst:") || line.startsWith("review failure burst:") ? "error" : "warning";
+}
+
+/** The worst (highest-severity) anomaly in a non-empty list, for the PagerDuty summary + severity — so a
+ *  repo with both a routine calibration nudge and an active-incident burst pages (if at all) at the burst's
+ *  urgency, not whichever anomaly happened to sort first. */
+export function worstAnomaly(anomalies: string[]): { line: string; severity: PagerDutySeverity } {
+  const severityRank: Record<PagerDutySeverity, number> = { info: 0, warning: 1, error: 2, critical: 3 };
+  let best = { line: anomalies[0] ?? "ops anomaly detected", severity: classifyAnomalySeverity(anomalies[0] ?? "") };
+  for (const line of anomalies) {
+    const severity = classifyAnomalySeverity(line);
+    if (severityRank[severity] > severityRank[best.severity]) best = { line, severity };
+  }
+  return best;
+}
+
 // ── Cron alerts: scan gittensory's outcome data, emit a structured log on drift (flag-gated by the caller) ──
 
 /** The registered repos to scan. Scoped to REGISTERED repos (the ones gittensory actually tracks outcomes
@@ -196,6 +220,27 @@ export async function runOpsAlerts(env: Env): Promise<Record<string, string[]>>
         // Structured log = gittensory's notify path (no Discord/operator webhook exists) AND the Sentry path
         // (level:"error" + an `event` field reaches forwardStructuredLogToSentry). One line per repo.
         console.error(JSON.stringify({ level: "error", event: "ops_anomaly", repo: repoFullName, at: nowIso(), anomalies }));
+        // Experimental PagerDuty paging (#4937): no-op unless GITTENSORY_ENABLE_PAGERDUTY is set AND a routing
+        // key resolves for this repo (resolvePagerDutyRoutingKey). ops_anomaly is this codebase's own existing
+        // "something needs a human" judgment call -- reusing it here (rather than paging on every
+        // captureError/captureReviewFailure call, which would need its own frequency/threshold policy first)
+        // keeps this narrow and low-risk. Pages at the WORST anomaly's severity; triggerPagerDutyIncident itself
+        // applies the min-severity floor (routine calibration nudges never page by default) and a cooldown (a
+        // still-ongoing anomaly across consecutive cron ticks does not re-page every tick) -- see its own
+        // comment for why alert fatigue needed both controls, not just PagerDuty's own dedup_key. Awaited (not
+        // fire-and-forget) so a page failure is captured within THIS tick's own error handling, not orphaned
+        // after runOpsAlerts has already returned -- triggerPagerDutyIncident itself never throws and bounds
+        // its own HTTP call to a 5s timeout, so this cannot hang the scan. This does not yet send a matching
+        // "resolve" event once anomalies clear (would need tracking previous-tick state) -- an operator
+        // currently resolves the incident manually once the underlying condition is fixed.
+        const worst = worstAnomaly(anomalies);
+        await triggerPagerDutyIncident(env, {
+          repoFullName,
+          summary: worst.line,
+          severity: worst.severity,
+          dedupKey: `ops_anomaly:${repoFullName}`,
+          customDetails: { anomalies },
+        });
         // #ops-anomaly-metric: Prometheus counterpart to the log line above so a self-host operator can alert on
         // /metrics instead of grepping Workers Logs. Scoped to reviewBurst/reviewFailureBurst -- the two anomalies
         // this module exists to catch fast (#orb-ci-stuck-repeat / #review-burst-blind-spot) -- rather than every