feat(governor): page PagerDuty on kill-switch trips

apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx

@@ -39,7 +39,11 @@ nothing should hold a stale lock.
 
 ## 1. Detection
 
-Flag a misbehaving loop from any of:
+Since #7666, a kill-switch **trip** (not a resume) also pages PagerDuty automatically — the same
+opt-in `LOOPOVER_ENABLE_PAGERDUTY` / `PAGERDUTY_ROUTING_KEY` contract as ORB's own alerting
+(`src/services/notify-pagerduty.ts`) — so the operator "on the page" below no longer has to already
+be watching a dashboard to notice the halt. Absent that, or as a second signal, flag a misbehaving
+loop from any of:
 
 1. Direct observation — destructive/off-scope file changes, a PR touching the wrong surface, repeated nonsensical commits.
 2. Soft-claim inventory — confirm what is in flight before you act:

packages/loopover-engine/src/governor/kill-switch.ts

@@ -73,3 +73,51 @@ export function buildMinerKillSwitchTransitionGovernorLedgerEvent(input: {
     payload: { previousScope: input.previousScope, scope: input.scope },
   };
 }
+
+/** Same literal set as ORB's hosted `PagerDutySeverity` (`src/services/notify-pagerduty.ts`) — kept as a local
+ *  literal union rather than importing that module, since it lives in the main app, not this shared package. */
+export type MinerKillSwitchPagerDutySeverity = "critical" | "error" | "warning" | "info";
+
+/** Pure PagerDuty alert payload for a kill-switch TRIP (#7666). Never built for a resume — clearing a halt is
+ *  relief, not an incident. */
+export type MinerKillSwitchPagerDutyAlert = {
+  repoFullName: string | null;
+  scope: MinerKillSwitchScope;
+  actionClass: string;
+  summary: string;
+  severity: MinerKillSwitchPagerDutySeverity;
+  dedupKey: string;
+  customDetails: Record<string, unknown>;
+};
+
+/**
+ * Build the PagerDuty alert payload for a kill-switch TRIP transition (#7666) — the paging counterpart to
+ * {@link buildMinerKillSwitchTransitionGovernorLedgerEvent}, sharing its exact "no-op unless the scope actually
+ * changed" gate, but narrower: it additionally returns `null` on a transition INTO `"none"` (a resume), since
+ * paging on "the halt cleared" would be noise, not an incident that needs a human. DETECTOR ONLY — no IO, same
+ * as this whole module: `packages/loopover-miner/lib/governor-kill-switch.ts` performs the actual PagerDuty
+ * Events API v2 call, mirroring how it (not this module) also performs the ledger IO for the sibling ledger-event
+ * builder above. `dedupKey` intentionally omits `actionClass` — a repo/scope kill-switch trip is one incident
+ * regardless of which action class first observed it, so PagerDuty's own dedup_key coalescing collapses repeats
+ * into the same incident instead of opening a new one per action class.
+ */
+export function buildMinerKillSwitchPagerDutyAlert(input: {
+  repoFullName?: string | null | undefined;
+  actionClass: string;
+  previousScope: MinerKillSwitchScope;
+  scope: MinerKillSwitchScope;
+}): MinerKillSwitchPagerDutyAlert | null {
+  if (input.previousScope === input.scope) return null;
+  if (!isMinerKillSwitchActive(input.scope)) return null;
+  const repoFullName = input.repoFullName ?? null;
+  const target = repoFullName ?? "global";
+  return {
+    repoFullName,
+    scope: input.scope,
+    actionClass: input.actionClass,
+    summary: `AMS miner kill-switch tripped (${input.scope}) — ${input.actionClass} halted for ${target}`,
+    severity: "critical",
+    dedupKey: `miner_kill_switch_tripped:${input.scope}:${target}`,
+    customDetails: { scope: input.scope, previousScope: input.previousScope, repoFullName, actionClass: input.actionClass },
+  };
+}

packages/loopover-engine/test/kill-switch.test.ts

@@ -3,6 +3,7 @@ import { test } from "node:test";
 
 import {
   MINER_KILL_SWITCH_ENV_VAR,
+  buildMinerKillSwitchPagerDutyAlert,
   buildMinerKillSwitchTransitionGovernorLedgerEvent,
   isGlobalMinerKillSwitch,
   isMinerKillSwitchActive,
@@ -14,6 +15,7 @@ test("barrel: the public entrypoint re-exports the kill-switch primitive (#2341)
   assert.equal(typeof resolveMinerKillSwitch, "function");
   assert.equal(typeof isMinerKillSwitchActive, "function");
   assert.equal(typeof buildMinerKillSwitchTransitionGovernorLedgerEvent, "function");
+  assert.equal(typeof buildMinerKillSwitchPagerDutyAlert, "function");
   assert.equal(MINER_KILL_SWITCH_ENV_VAR, "LOOPOVER_MINER_KILL_SWITCH");
 });
 
@@ -105,3 +107,65 @@ test("buildMinerKillSwitchTransitionGovernorLedgerEvent: clearing the switch rec
     payload: { previousScope: "global", scope: "none" },
   });
 });
+
+test("buildMinerKillSwitchPagerDutyAlert: no-op when the scope has not changed (#7666)", () => {
+  assert.equal(
+    buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "none", scope: "none" }),
+    null,
+  );
+  assert.equal(
+    buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "repo", scope: "repo" }),
+    null,
+  );
+});
+
+test("buildMinerKillSwitchPagerDutyAlert: no-op on a resume transition -- only a trip pages (#7666)", () => {
+  assert.equal(
+    buildMinerKillSwitchPagerDutyAlert({
+      repoFullName: "acme/widgets",
+      actionClass: "open_pr",
+      previousScope: "repo",
+      scope: "none",
+    }),
+    null,
+  );
+  assert.equal(
+    buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "global", scope: "none" }),
+    null,
+  );
+});
+
+test("buildMinerKillSwitchPagerDutyAlert: a repo trip builds a critical alert with a repo-scoped dedup key (#7666)", () => {
+  const alert = buildMinerKillSwitchPagerDutyAlert({
+    repoFullName: "acme/widgets",
+    actionClass: "open_pr",
+    previousScope: "none",
+    scope: "repo",
+  });
+  assert.deepEqual(alert, {
+    repoFullName: "acme/widgets",
+    scope: "repo",
+    actionClass: "open_pr",
+    summary: "AMS miner kill-switch tripped (repo) — open_pr halted for acme/widgets",
+    severity: "critical",
+    dedupKey: "miner_kill_switch_tripped:repo:acme/widgets",
+    customDetails: { scope: "repo", previousScope: "none", repoFullName: "acme/widgets", actionClass: "open_pr" },
+  });
+});
+
+test("buildMinerKillSwitchPagerDutyAlert: a global trip with no repoFullName dedups on 'global', not null (#7666)", () => {
+  const alert = buildMinerKillSwitchPagerDutyAlert({
+    actionClass: "open_pr",
+    previousScope: "none",
+    scope: "global",
+  });
+  assert.deepEqual(alert, {
+    repoFullName: null,
+    scope: "global",
+    actionClass: "open_pr",
+    summary: "AMS miner kill-switch tripped (global) — open_pr halted for global",
+    severity: "critical",
+    dedupKey: "miner_kill_switch_tripped:global:global",
+    customDetails: { scope: "global", previousScope: "none", repoFullName: null, actionClass: "open_pr" },
+  });
+});

packages/loopover-miner/lib/governor-kill-switch.ts

@@ -2,17 +2,86 @@
 // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the
 // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed
 // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence.
+//
+// PagerDuty paging (#7666): a TRIP transition also fires a page, mirroring ORB's hosted `triggerPagerDutyIncident`
+// (src/services/notify-pagerduty.ts) Events API v2 contract -- same LOOPOVER_ENABLE_PAGERDUTY flag, same
+// PAGERDUTY_ROUTING_KEY, same enqueue URL/payload shape -- with the same simplification #7667's control-plane
+// mirror (control-plane/src/pagerduty-notify.ts) used: no D1/Worker Env here either (the miner is a plain Node
+// process), so no per-repo routing-key map and no severity-threshold/cooldown DB query; PagerDuty's own
+// `dedup_key` still coalesces duplicate incidents. Best-effort: paging can never block or throw past the ledger
+// write it accompanies.
 
 import {
+  buildMinerKillSwitchPagerDutyAlert,
   buildMinerKillSwitchTransitionGovernorLedgerEvent,
   isGlobalMinerKillSwitch,
   isMinerKillSwitchActive,
   resolveMinerKillSwitch,
 } from "@loopover/engine";
-import type { MinerKillSwitchScope } from "@loopover/engine";
+import type { MinerKillSwitchPagerDutyAlert, MinerKillSwitchScope } from "@loopover/engine";
 import { appendGovernorEvent } from "./governor-ledger.js";
 import type { AppendGovernorEventInput, GovernorLedgerEntry } from "./governor-ledger.js";
 
+const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue";
+// PagerDuty routing/integration keys are 32 lowercase hex characters.
+const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i;
+const TRUTHY_ENV = /^(1|true|yes|on)$/i;
+
+export type NotifyMinerKillSwitchPagerDuty = (
+  alert: MinerKillSwitchPagerDutyAlert,
+  env: Record<string, string | undefined>,
+) => void | Promise<void>;
+
+function envString(env: Record<string, string | undefined>, name: string): string | undefined {
+  const value = env[name];
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+}
+
+function warnMinerKillSwitchPagerDutyFailed(dedupKey: string, error: unknown): void {
+  const message = (error instanceof Error ? error.message : String(error)).slice(0, 200);
+  console.warn(JSON.stringify({ event: "miner_kill_switch_pagerduty_failed", dedupKey, message }));
+}
+
+/** Miner-side mirror of ORB's `triggerPagerDutyIncident` (src/services/notify-pagerduty.ts) Events API v2
+ *  contract, same simplification #7667's control-plane mirror used (no D1/Worker Env here either): same
+ *  LOOPOVER_ENABLE_PAGERDUTY flag, same global PAGERDUTY_ROUTING_KEY, same enqueue URL/payload shape. PagerDuty's
+ *  own dedup_key still coalesces duplicate incidents. Best-effort: never throws -- a paging failure must never
+ *  block or mask the governor ledger write it is reporting on. */
+export async function notifyMinerKillSwitchPagerDuty(
+  alert: MinerKillSwitchPagerDutyAlert,
+  env: Record<string, string | undefined> = process.env,
+): Promise<void> {
+  if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) return;
+  const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY");
+  if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) return;
+
+  try {
+    const response = await fetch(PAGERDUTY_EVENTS_URL, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        routing_key: routingKey,
+        event_action: "trigger",
+        dedup_key: alert.dedupKey,
+        payload: {
+          summary: alert.summary.slice(0, 1024),
+          source: "loopover-miner",
+          severity: alert.severity,
+          timestamp: new Date().toISOString(),
+          component: alert.repoFullName ?? "global",
+          custom_details: alert.customDetails,
+        },
+      }),
+      signal: AbortSignal.timeout(5000),
+    });
+    if (!response.ok) {
+      console.warn(JSON.stringify({ event: "miner_kill_switch_pagerduty_failed", dedupKey: alert.dedupKey, status: response.status }));
+    }
+  } catch (error) {
+    warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error);
+  }
+}
+
 export type CheckMinerKillSwitchInput = {
   repoPaused?: boolean;
   env?: Record<string, string | undefined>;
@@ -41,17 +110,46 @@ export type RecordMinerKillSwitchTransitionInput = {
   scope: MinerKillSwitchScope;
 };
 
+export type RecordMinerKillSwitchTransitionOptions = {
+  append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry;
+  /** Injectable for tests; defaults to the real {@link notifyMinerKillSwitchPagerDuty} Events API v2 call. */
+  notify?: NotifyMinerKillSwitchPagerDuty;
+  /** Defaults to `process.env`, matching {@link notifyMinerKillSwitchPagerDuty}'s own default. */
+  env?: Record<string, string | undefined>;
+};
+
 /**
  * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the
  * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory
  * or persisted); this module holds no state of its own.
+ *
+ * On a TRIP (not a resume), also pages PagerDuty (#7666) via {@link notifyMinerKillSwitchPagerDuty}: the ledger
+ * row is appended FIRST, then paging is fired fire-and-forget (wrapped in both a sync try/catch and a `.catch`
+ * on its returned promise, so neither a synchronous throw nor an async rejection from the notify hook can ever
+ * block or mask the ledger write that already landed).
  */
 export function recordMinerKillSwitchTransition(
   input: RecordMinerKillSwitchTransitionInput,
-  options: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry } = {},
+  options: RecordMinerKillSwitchTransitionOptions = {},
 ): GovernorLedgerEntry | null {
   const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input);
   if (!event) return null;
   const append = options.append ?? appendGovernorEvent;
-  return append(event as AppendGovernorEventInput);
+  const entry = append(event as AppendGovernorEventInput);
+
+  const alert = buildMinerKillSwitchPagerDutyAlert(input);
+  if (alert) {
+    const notify = options.notify ?? notifyMinerKillSwitchPagerDuty;
+    const env = options.env ?? process.env;
+    try {
+      const result = notify(alert, env);
+      if (result && typeof (result as Promise<void>).catch === "function") {
+        (result as Promise<void>).catch((error: unknown) => warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error));
+      }
+    } catch (error) {
+      warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error);
+    }
+  }
+
+  return entry;
 }

test/unit/governor-kill-switch-pagerduty.test.ts

@@ -0,0 +1,75 @@
+// Pure-builder tests for buildMinerKillSwitchPagerDutyAlert (#7666) -- the PagerDuty-paging counterpart to
+// buildMinerKillSwitchTransitionGovernorLedgerEvent (see governor-run-halt.test.ts / kill-switch-incident-
+// runbook.test.ts for the same "test the engine's pure calculator directly" convention). The IO wrapper that
+// actually fires the Events API v2 call (packages/loopover-miner/lib/governor-kill-switch.ts's
+// notifyMinerKillSwitchPagerDuty) is covered by test/unit/miner-governor-kill-switch.test.ts.
+import { describe, expect, it } from "vitest";
+import { buildMinerKillSwitchPagerDutyAlert } from "../../packages/loopover-engine/src/governor/kill-switch";
+
+describe("buildMinerKillSwitchPagerDutyAlert (#7666)", () => {
+  it("no-op when the scope has not changed", () => {
+    expect(buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "none", scope: "none" })).toBeNull();
+    expect(buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "repo", scope: "repo" })).toBeNull();
+    expect(buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "global", scope: "global" })).toBeNull();
+  });
+
+  it("no-op on a resume transition (a transition INTO 'none') -- only a trip pages, never a resume", () => {
+    expect(
+      buildMinerKillSwitchPagerDutyAlert({
+        repoFullName: "acme/widgets",
+        actionClass: "open_pr",
+        previousScope: "repo",
+        scope: "none",
+      }),
+    ).toBeNull();
+    expect(buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "global", scope: "none" })).toBeNull();
+  });
+
+  it("a repo trip builds a critical alert with a repo-scoped dedup key and component", () => {
+    const alert = buildMinerKillSwitchPagerDutyAlert({
+      repoFullName: "acme/widgets",
+      actionClass: "open_pr",
+      previousScope: "none",
+      scope: "repo",
+    });
+    expect(alert).toEqual({
+      repoFullName: "acme/widgets",
+      scope: "repo",
+      actionClass: "open_pr",
+      summary: "AMS miner kill-switch tripped (repo) — open_pr halted for acme/widgets",
+      severity: "critical",
+      dedupKey: "miner_kill_switch_tripped:repo:acme/widgets",
+      customDetails: { scope: "repo", previousScope: "none", repoFullName: "acme/widgets", actionClass: "open_pr" },
+    });
+  });
+
+  it("a global trip with no repoFullName supplied dedups/reports on the literal 'global' target, not null or omitted", () => {
+    const alert = buildMinerKillSwitchPagerDutyAlert({
+      actionClass: "open_pr",
+      previousScope: "none",
+      scope: "global",
+    });
+    expect(alert).toEqual({
+      repoFullName: null,
+      scope: "global",
+      actionClass: "open_pr",
+      summary: "AMS miner kill-switch tripped (global) — open_pr halted for global",
+      severity: "critical",
+      dedupKey: "miner_kill_switch_tripped:global:global",
+      customDetails: { scope: "global", previousScope: "none", repoFullName: null, actionClass: "open_pr" },
+    });
+  });
+
+  it("REGRESSION: a global trip that also carries a repoFullName still dedups per-repo, matching component", () => {
+    // Not a real-world combination (global halts every repo at once) but the builder must not silently drop
+    // an unexpectedly-present repoFullName -- it should behave identically to the repo-scope case for targeting.
+    const alert = buildMinerKillSwitchPagerDutyAlert({
+      repoFullName: "acme/widgets",
+      actionClass: "open_pr",
+      previousScope: "none",
+      scope: "global",
+    });
+    expect(alert?.dedupKey).toBe("miner_kill_switch_tripped:global:acme/widgets");
+    expect(alert?.repoFullName).toBe("acme/widgets");
+  });
+});