feat(ams): page PagerDuty on miner kill-switch trips

apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx

@@ -52,6 +52,13 @@ Flag a misbehaving loop from any of:
 3. Fleet observability — error-rate spikes, abnormal claim/submission patterns, elevated
    `consecutive_failures` / rejection-reason clustering, or an explicit customer/support escalation
    (see the rented-loop escalation path from #4806).
+4. **Automated PagerDuty page** (#7666) — when `LOOPOVER_ENABLE_PAGERDUTY` is on and
+   `PAGERDUTY_ROUTING_KEY` is configured in the **miner** process, a kill-switch **trip** (engage)
+   fires an Events API v2 page using the same flag / routing-key / enqueue contract as ORB's
+   `notify-pagerduty` module. A resume does not page. The miner has no D1-backed severity floor or
+   cooldown (PagerDuty's own `dedup_key` still coalesces duplicate incidents). Treat an
+   `ams_kill_switch:*` incident as "operator already needed on the page" and start the 15-minute
+   response clock immediately.
 
 **Triage before acting:** decide one-repo vs fleet-wide. Prefer the narrowest scope that stops the
 harm so one bad tenant does not force an unnecessary global halt.

packages/loopover-engine/src/governor/kill-switch.ts

@@ -7,7 +7,9 @@
 //
 // DETECTOR ONLY — no IO, no persistence. Composing this with the other pure calculators into one fail-closed
 // allow/deny verdict (and recording every CHECK, not just a transition) is the Governor chokepoint's job
-// (#2340), which consults this module first in its "safest wins" precedence.
+// (#2340), which consults this module first in its "safest wins" precedence. Paging on a trip (#7666) is also
+// an IO concern: this module only builds the pure PagerDuty alert payload; the miner IO seam fires the
+// Events API call (mirroring `src/services/notify-pagerduty.ts`'s contract — AMS trips have no hosted path).
 
 import type { GovernorLedgerEvent } from "../governor-ledger.js";
 
@@ -73,3 +75,43 @@ export function buildMinerKillSwitchTransitionGovernorLedgerEvent(input: {
     payload: { previousScope: input.previousScope, scope: input.scope },
   };
 }
+
+/**
+ * Pure page payload for a kill-switch TRIP (#7666). Returns `null` when the transition is not a trip
+ * (no-op same-scope, or a resume) — paging wakes humans for engage, not for clear. `repoFullName` falls
+ * back to `ams/fleet` for a global halt with no single-repo context so routing still resolves against the
+ * operator's global PagerDuty key. Consumers (miner IO seam) own the actual Events API call — there is no
+ * hosted AMS trip path today, so this module stays detector-only.
+ */
+export type MinerKillSwitchPagerDutyAlert = {
+  repoFullName: string;
+  summary: string;
+  severity: "critical";
+  dedupKey: string;
+  customDetails: {
+    previousScope: MinerKillSwitchScope;
+    scope: MinerKillSwitchScope;
+    reason: string;
+  };
+};
+
+export function buildMinerKillSwitchPagerDutyAlert(input: {
+  repoFullName?: string | null | undefined;
+  previousScope: MinerKillSwitchScope;
+  scope: MinerKillSwitchScope;
+}): MinerKillSwitchPagerDutyAlert | null {
+  if (input.previousScope === input.scope) return null;
+  if (!isMinerKillSwitchActive(input.scope)) return null;
+  const repoFullName = (input.repoFullName ?? "").trim() || "ams/fleet";
+  const reason = `${input.scope}_kill_switch_engaged`;
+  return {
+    repoFullName,
+    summary:
+      input.scope === "global"
+        ? `AMS miner kill-switch engaged (global / fleet-wide)`
+        : `AMS miner kill-switch engaged (repo) for ${repoFullName}`,
+    severity: "critical",
+    dedupKey: `ams_kill_switch:${input.scope}:${repoFullName.toLowerCase()}`,
+    customDetails: { previousScope: input.previousScope, scope: input.scope, reason },
+  };
+}

packages/loopover-engine/test/kill-switch.test.ts

@@ -3,6 +3,7 @@ import { test } from "node:test";
 
 import {
   MINER_KILL_SWITCH_ENV_VAR,
+  buildMinerKillSwitchPagerDutyAlert,
   buildMinerKillSwitchTransitionGovernorLedgerEvent,
   isGlobalMinerKillSwitch,
   isMinerKillSwitchActive,
@@ -14,6 +15,7 @@ test("barrel: the public entrypoint re-exports the kill-switch primitive (#2341)
   assert.equal(typeof resolveMinerKillSwitch, "function");
   assert.equal(typeof isMinerKillSwitchActive, "function");
   assert.equal(typeof buildMinerKillSwitchTransitionGovernorLedgerEvent, "function");
+  assert.equal(typeof buildMinerKillSwitchPagerDutyAlert, "function");
   assert.equal(MINER_KILL_SWITCH_ENV_VAR, "LOOPOVER_MINER_KILL_SWITCH");
 });
 
@@ -105,3 +107,46 @@ test("buildMinerKillSwitchTransitionGovernorLedgerEvent: clearing the switch rec
     payload: { previousScope: "global", scope: "none" },
   });
 });
+
+test("buildMinerKillSwitchPagerDutyAlert: trip builds a critical page payload (#7666)", () => {
+  const alert = buildMinerKillSwitchPagerDutyAlert({
+    repoFullName: "acme/widgets",
+    previousScope: "none",
+    scope: "repo",
+  });
+  assert.deepEqual(alert, {
+    repoFullName: "acme/widgets",
+    summary: "AMS miner kill-switch engaged (repo) for acme/widgets",
+    severity: "critical",
+    dedupKey: "ams_kill_switch:repo:acme/widgets",
+    customDetails: { previousScope: "none", scope: "repo", reason: "repo_kill_switch_engaged" },
+  });
+});
+
+test("buildMinerKillSwitchPagerDutyAlert: global trip without a repo uses ams/fleet (#7666)", () => {
+  const alert = buildMinerKillSwitchPagerDutyAlert({
+    previousScope: "none",
+    scope: "global",
+  });
+  assert.equal(alert?.repoFullName, "ams/fleet");
+  assert.equal(alert?.dedupKey, "ams_kill_switch:global:ams/fleet");
+  assert.match(alert?.summary ?? "", /fleet-wide/);
+
+  const blankRepo = buildMinerKillSwitchPagerDutyAlert({
+    repoFullName: "   ",
+    previousScope: "none",
+    scope: "global",
+  });
+  assert.equal(blankRepo?.repoFullName, "ams/fleet");
+});
+
+test("buildMinerKillSwitchPagerDutyAlert: resume / same-scope are silent (#7666)", () => {
+  assert.equal(
+    buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "none" }),
+    null,
+  );
+  assert.equal(
+    buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "repo" }),
+    null,
+  );
+});

packages/loopover-miner/lib/governor-kill-switch.d.ts

@@ -1,3 +1,4 @@
+import { type MinerKillSwitchPagerDutyAlert } from "@loopover/engine";
 import type { MinerKillSwitchScope } from "@loopover/engine";
 import type { AppendGovernorEventInput, GovernorLedgerEntry } from "./governor-ledger.js";
 export type CheckMinerKillSwitchInput = {
@@ -19,11 +20,21 @@ export type RecordMinerKillSwitchTransitionInput = {
     previousScope: MinerKillSwitchScope;
     scope: MinerKillSwitchScope;
 };
+export type NotifyMinerKillSwitchTrip = (alert: MinerKillSwitchPagerDutyAlert, env: Record<string, string | undefined>) => void | Promise<void>;
+/**
+ * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events
+ * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still
+ * coalesces duplicate incidents. Best-effort: never throws.
+ */
+export declare function notifyMinerKillSwitchPagerDuty(alert: MinerKillSwitchPagerDutyAlert, env?: Record<string, string | undefined>): Promise<void>;
 /**
  * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the
- * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory
- * or persisted); this module holds no state of its own.
+ * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory
+ * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666)
+ * unless `notify` is overridden (tests) or the integration flag/key is unset.
  */
 export declare function recordMinerKillSwitchTransition(input: RecordMinerKillSwitchTransitionInput, options?: {
     append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry;
+    notify?: NotifyMinerKillSwitchTrip;
+    env?: Record<string, string | undefined>;
 }): GovernorLedgerEntry | null;