From 78debc29d9ae5996e09f90fc8aabfaac26c8032f Mon Sep 17 00:00:00 2001 From: galuis116 Date: Wed, 22 Jul 2026 11:55:09 -0400 Subject: [PATCH] feat(governor): page PagerDuty on kill-switch trips Wire the miner AMS kill-switch trip path into the existing PagerDuty alerting pattern instead of a new mechanism. packages/loopover-engine's kill-switch.ts gains a pure buildMinerKillSwitchPagerDutyAlert builder (mirrors buildMinerKillSwitchTransitionGovernorLedgerEvent's own no-op-unless-changed gate, but only on a TRIP, never a resume). packages/loopover-miner's governor-kill-switch.ts gains the IO wrapper notifyMinerKillSwitchPagerDuty, mirroring src/services/notify-pagerduty.ts's Events API v2 contract (LOOPOVER_ENABLE_PAGERDUTY flag, PAGERDUTY_ROUTING_KEY, dedup_key) with the same no-D1/Worker-Env simplification control-plane's own mirror (#7667) used. recordMinerKillSwitchTransition now pages fire-and-forget after the ledger row lands, wrapped so a paging failure can never block or mask the ledger write. Closes #7666 --- .../content/docs/ams-kill-switch-incident.mdx | 6 +- .../src/governor/kill-switch.ts | 48 ++++ .../loopover-engine/test/kill-switch.test.ts | 64 +++++ .../lib/governor-kill-switch.ts | 104 +++++++- .../governor-kill-switch-pagerduty.test.ts | 75 ++++++ test/unit/miner-governor-kill-switch.test.ts | 248 +++++++++++++++++- 6 files changed, 540 insertions(+), 5 deletions(-) create mode 100644 test/unit/governor-kill-switch-pagerduty.test.ts diff --git a/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx b/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx index 90a9bd9289..2326544be7 100644 --- a/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx +++ b/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx @@ -39,7 +39,11 @@ nothing should hold a stale lock. ## 1. Detection -Flag a misbehaving loop from any of: +Since #7666, a kill-switch **trip** (not a resume) also pages PagerDuty automatically — the same +opt-in `LOOPOVER_ENABLE_PAGERDUTY` / `PAGERDUTY_ROUTING_KEY` contract as ORB's own alerting +(`src/services/notify-pagerduty.ts`) — so the operator "on the page" below no longer has to already +be watching a dashboard to notice the halt. Absent that, or as a second signal, flag a misbehaving +loop from any of: 1. Direct observation — destructive/off-scope file changes, a PR touching the wrong surface, repeated nonsensical commits. 2. Soft-claim inventory — confirm what is in flight before you act: diff --git a/packages/loopover-engine/src/governor/kill-switch.ts b/packages/loopover-engine/src/governor/kill-switch.ts index 1f49f119e9..b6d0b5f8d1 100644 --- a/packages/loopover-engine/src/governor/kill-switch.ts +++ b/packages/loopover-engine/src/governor/kill-switch.ts @@ -73,3 +73,51 @@ export function buildMinerKillSwitchTransitionGovernorLedgerEvent(input: { payload: { previousScope: input.previousScope, scope: input.scope }, }; } + +/** Same literal set as ORB's hosted `PagerDutySeverity` (`src/services/notify-pagerduty.ts`) — kept as a local + * literal union rather than importing that module, since it lives in the main app, not this shared package. */ +export type MinerKillSwitchPagerDutySeverity = "critical" | "error" | "warning" | "info"; + +/** Pure PagerDuty alert payload for a kill-switch TRIP (#7666). Never built for a resume — clearing a halt is + * relief, not an incident. */ +export type MinerKillSwitchPagerDutyAlert = { + repoFullName: string | null; + scope: MinerKillSwitchScope; + actionClass: string; + summary: string; + severity: MinerKillSwitchPagerDutySeverity; + dedupKey: string; + customDetails: Record; +}; + +/** + * Build the PagerDuty alert payload for a kill-switch TRIP transition (#7666) — the paging counterpart to + * {@link buildMinerKillSwitchTransitionGovernorLedgerEvent}, sharing its exact "no-op unless the scope actually + * changed" gate, but narrower: it additionally returns `null` on a transition INTO `"none"` (a resume), since + * paging on "the halt cleared" would be noise, not an incident that needs a human. DETECTOR ONLY — no IO, same + * as this whole module: `packages/loopover-miner/lib/governor-kill-switch.ts` performs the actual PagerDuty + * Events API v2 call, mirroring how it (not this module) also performs the ledger IO for the sibling ledger-event + * builder above. `dedupKey` intentionally omits `actionClass` — a repo/scope kill-switch trip is one incident + * regardless of which action class first observed it, so PagerDuty's own dedup_key coalescing collapses repeats + * into the same incident instead of opening a new one per action class. + */ +export function buildMinerKillSwitchPagerDutyAlert(input: { + repoFullName?: string | null | undefined; + actionClass: string; + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; +}): MinerKillSwitchPagerDutyAlert | null { + if (input.previousScope === input.scope) return null; + if (!isMinerKillSwitchActive(input.scope)) return null; + const repoFullName = input.repoFullName ?? null; + const target = repoFullName ?? "global"; + return { + repoFullName, + scope: input.scope, + actionClass: input.actionClass, + summary: `AMS miner kill-switch tripped (${input.scope}) — ${input.actionClass} halted for ${target}`, + severity: "critical", + dedupKey: `miner_kill_switch_tripped:${input.scope}:${target}`, + customDetails: { scope: input.scope, previousScope: input.previousScope, repoFullName, actionClass: input.actionClass }, + }; +} diff --git a/packages/loopover-engine/test/kill-switch.test.ts b/packages/loopover-engine/test/kill-switch.test.ts index b06e5a5c15..3fc20f69f4 100644 --- a/packages/loopover-engine/test/kill-switch.test.ts +++ b/packages/loopover-engine/test/kill-switch.test.ts @@ -3,6 +3,7 @@ import { test } from "node:test"; import { MINER_KILL_SWITCH_ENV_VAR, + buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, @@ -14,6 +15,7 @@ test("barrel: the public entrypoint re-exports the kill-switch primitive (#2341) assert.equal(typeof resolveMinerKillSwitch, "function"); assert.equal(typeof isMinerKillSwitchActive, "function"); assert.equal(typeof buildMinerKillSwitchTransitionGovernorLedgerEvent, "function"); + assert.equal(typeof buildMinerKillSwitchPagerDutyAlert, "function"); assert.equal(MINER_KILL_SWITCH_ENV_VAR, "LOOPOVER_MINER_KILL_SWITCH"); }); @@ -105,3 +107,65 @@ test("buildMinerKillSwitchTransitionGovernorLedgerEvent: clearing the switch rec payload: { previousScope: "global", scope: "none" }, }); }); + +test("buildMinerKillSwitchPagerDutyAlert: no-op when the scope has not changed (#7666)", () => { + assert.equal( + buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "none", scope: "none" }), + null, + ); + assert.equal( + buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "repo", scope: "repo" }), + null, + ); +}); + +test("buildMinerKillSwitchPagerDutyAlert: no-op on a resume transition -- only a trip pages (#7666)", () => { + assert.equal( + buildMinerKillSwitchPagerDutyAlert({ + repoFullName: "acme/widgets", + actionClass: "open_pr", + previousScope: "repo", + scope: "none", + }), + null, + ); + assert.equal( + buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "global", scope: "none" }), + null, + ); +}); + +test("buildMinerKillSwitchPagerDutyAlert: a repo trip builds a critical alert with a repo-scoped dedup key (#7666)", () => { + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: "acme/widgets", + actionClass: "open_pr", + previousScope: "none", + scope: "repo", + }); + assert.deepEqual(alert, { + repoFullName: "acme/widgets", + scope: "repo", + actionClass: "open_pr", + summary: "AMS miner kill-switch tripped (repo) — open_pr halted for acme/widgets", + severity: "critical", + dedupKey: "miner_kill_switch_tripped:repo:acme/widgets", + customDetails: { scope: "repo", previousScope: "none", repoFullName: "acme/widgets", actionClass: "open_pr" }, + }); +}); + +test("buildMinerKillSwitchPagerDutyAlert: a global trip with no repoFullName dedups on 'global', not null (#7666)", () => { + const alert = buildMinerKillSwitchPagerDutyAlert({ + actionClass: "open_pr", + previousScope: "none", + scope: "global", + }); + assert.deepEqual(alert, { + repoFullName: null, + scope: "global", + actionClass: "open_pr", + summary: "AMS miner kill-switch tripped (global) — open_pr halted for global", + severity: "critical", + dedupKey: "miner_kill_switch_tripped:global:global", + customDetails: { scope: "global", previousScope: "none", repoFullName: null, actionClass: "open_pr" }, + }); +}); diff --git a/packages/loopover-miner/lib/governor-kill-switch.ts b/packages/loopover-miner/lib/governor-kill-switch.ts index 71a4de6773..4f73711f35 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.ts +++ b/packages/loopover-miner/lib/governor-kill-switch.ts @@ -2,17 +2,86 @@ // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence. +// +// PagerDuty paging (#7666): a TRIP transition also fires a page, mirroring ORB's hosted `triggerPagerDutyIncident` +// (src/services/notify-pagerduty.ts) Events API v2 contract -- same LOOPOVER_ENABLE_PAGERDUTY flag, same +// PAGERDUTY_ROUTING_KEY, same enqueue URL/payload shape -- with the same simplification #7667's control-plane +// mirror (control-plane/src/pagerduty-notify.ts) used: no D1/Worker Env here either (the miner is a plain Node +// process), so no per-repo routing-key map and no severity-threshold/cooldown DB query; PagerDuty's own +// `dedup_key` still coalesces duplicate incidents. Best-effort: paging can never block or throw past the ledger +// write it accompanies. import { + buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, resolveMinerKillSwitch, } from "@loopover/engine"; -import type { MinerKillSwitchScope } from "@loopover/engine"; +import type { MinerKillSwitchPagerDutyAlert, MinerKillSwitchScope } from "@loopover/engine"; import { appendGovernorEvent } from "./governor-ledger.js"; import type { AppendGovernorEventInput, GovernorLedgerEntry } from "./governor-ledger.js"; +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; +// PagerDuty routing/integration keys are 32 lowercase hex characters. +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; +const TRUTHY_ENV = /^(1|true|yes|on)$/i; + +export type NotifyMinerKillSwitchPagerDuty = ( + alert: MinerKillSwitchPagerDutyAlert, + env: Record, +) => void | Promise; + +function envString(env: Record, name: string): string | undefined { + const value = env[name]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} + +function warnMinerKillSwitchPagerDutyFailed(dedupKey: string, error: unknown): void { + const message = (error instanceof Error ? error.message : String(error)).slice(0, 200); + console.warn(JSON.stringify({ event: "miner_kill_switch_pagerduty_failed", dedupKey, message })); +} + +/** Miner-side mirror of ORB's `triggerPagerDutyIncident` (src/services/notify-pagerduty.ts) Events API v2 + * contract, same simplification #7667's control-plane mirror used (no D1/Worker Env here either): same + * LOOPOVER_ENABLE_PAGERDUTY flag, same global PAGERDUTY_ROUTING_KEY, same enqueue URL/payload shape. PagerDuty's + * own dedup_key still coalesces duplicate incidents. Best-effort: never throws -- a paging failure must never + * block or mask the governor ledger write it is reporting on. */ +export async function notifyMinerKillSwitchPagerDuty( + alert: MinerKillSwitchPagerDutyAlert, + env: Record = process.env, +): Promise { + if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) return; + const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY"); + if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) return; + + try { + const response = await fetch(PAGERDUTY_EVENTS_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: routingKey, + event_action: "trigger", + dedup_key: alert.dedupKey, + payload: { + summary: alert.summary.slice(0, 1024), + source: "loopover-miner", + severity: alert.severity, + timestamp: new Date().toISOString(), + component: alert.repoFullName ?? "global", + custom_details: alert.customDetails, + }, + }), + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) { + console.warn(JSON.stringify({ event: "miner_kill_switch_pagerduty_failed", dedupKey: alert.dedupKey, status: response.status })); + } + } catch (error) { + warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error); + } +} + export type CheckMinerKillSwitchInput = { repoPaused?: boolean; env?: Record; @@ -41,17 +110,46 @@ export type RecordMinerKillSwitchTransitionInput = { scope: MinerKillSwitchScope; }; +export type RecordMinerKillSwitchTransitionOptions = { + append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry; + /** Injectable for tests; defaults to the real {@link notifyMinerKillSwitchPagerDuty} Events API v2 call. */ + notify?: NotifyMinerKillSwitchPagerDuty; + /** Defaults to `process.env`, matching {@link notifyMinerKillSwitchPagerDuty}'s own default. */ + env?: Record; +}; + /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory * or persisted); this module holds no state of its own. + * + * On a TRIP (not a resume), also pages PagerDuty (#7666) via {@link notifyMinerKillSwitchPagerDuty}: the ledger + * row is appended FIRST, then paging is fired fire-and-forget (wrapped in both a sync try/catch and a `.catch` + * on its returned promise, so neither a synchronous throw nor an async rejection from the notify hook can ever + * block or mask the ledger write that already landed). */ export function recordMinerKillSwitchTransition( input: RecordMinerKillSwitchTransitionInput, - options: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry } = {}, + options: RecordMinerKillSwitchTransitionOptions = {}, ): GovernorLedgerEntry | null { const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input); if (!event) return null; const append = options.append ?? appendGovernorEvent; - return append(event as AppendGovernorEventInput); + const entry = append(event as AppendGovernorEventInput); + + const alert = buildMinerKillSwitchPagerDutyAlert(input); + if (alert) { + const notify = options.notify ?? notifyMinerKillSwitchPagerDuty; + const env = options.env ?? process.env; + try { + const result = notify(alert, env); + if (result && typeof (result as Promise).catch === "function") { + (result as Promise).catch((error: unknown) => warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error)); + } + } catch (error) { + warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error); + } + } + + return entry; } diff --git a/test/unit/governor-kill-switch-pagerduty.test.ts b/test/unit/governor-kill-switch-pagerduty.test.ts new file mode 100644 index 0000000000..743735ae18 --- /dev/null +++ b/test/unit/governor-kill-switch-pagerduty.test.ts @@ -0,0 +1,75 @@ +// Pure-builder tests for buildMinerKillSwitchPagerDutyAlert (#7666) -- the PagerDuty-paging counterpart to +// buildMinerKillSwitchTransitionGovernorLedgerEvent (see governor-run-halt.test.ts / kill-switch-incident- +// runbook.test.ts for the same "test the engine's pure calculator directly" convention). The IO wrapper that +// actually fires the Events API v2 call (packages/loopover-miner/lib/governor-kill-switch.ts's +// notifyMinerKillSwitchPagerDuty) is covered by test/unit/miner-governor-kill-switch.test.ts. +import { describe, expect, it } from "vitest"; +import { buildMinerKillSwitchPagerDutyAlert } from "../../packages/loopover-engine/src/governor/kill-switch"; + +describe("buildMinerKillSwitchPagerDutyAlert (#7666)", () => { + it("no-op when the scope has not changed", () => { + expect(buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "none", scope: "none" })).toBeNull(); + expect(buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "repo", scope: "repo" })).toBeNull(); + expect(buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "global", scope: "global" })).toBeNull(); + }); + + it("no-op on a resume transition (a transition INTO 'none') -- only a trip pages, never a resume", () => { + expect( + buildMinerKillSwitchPagerDutyAlert({ + repoFullName: "acme/widgets", + actionClass: "open_pr", + previousScope: "repo", + scope: "none", + }), + ).toBeNull(); + expect(buildMinerKillSwitchPagerDutyAlert({ actionClass: "open_pr", previousScope: "global", scope: "none" })).toBeNull(); + }); + + it("a repo trip builds a critical alert with a repo-scoped dedup key and component", () => { + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: "acme/widgets", + actionClass: "open_pr", + previousScope: "none", + scope: "repo", + }); + expect(alert).toEqual({ + repoFullName: "acme/widgets", + scope: "repo", + actionClass: "open_pr", + summary: "AMS miner kill-switch tripped (repo) — open_pr halted for acme/widgets", + severity: "critical", + dedupKey: "miner_kill_switch_tripped:repo:acme/widgets", + customDetails: { scope: "repo", previousScope: "none", repoFullName: "acme/widgets", actionClass: "open_pr" }, + }); + }); + + it("a global trip with no repoFullName supplied dedups/reports on the literal 'global' target, not null or omitted", () => { + const alert = buildMinerKillSwitchPagerDutyAlert({ + actionClass: "open_pr", + previousScope: "none", + scope: "global", + }); + expect(alert).toEqual({ + repoFullName: null, + scope: "global", + actionClass: "open_pr", + summary: "AMS miner kill-switch tripped (global) — open_pr halted for global", + severity: "critical", + dedupKey: "miner_kill_switch_tripped:global:global", + customDetails: { scope: "global", previousScope: "none", repoFullName: null, actionClass: "open_pr" }, + }); + }); + + it("REGRESSION: a global trip that also carries a repoFullName still dedups per-repo, matching component", () => { + // Not a real-world combination (global halts every repo at once) but the builder must not silently drop + // an unexpectedly-present repoFullName -- it should behave identically to the repo-scope case for targeting. + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: "acme/widgets", + actionClass: "open_pr", + previousScope: "none", + scope: "global", + }); + expect(alert?.dedupKey).toBe("miner_kill_switch_tripped:global:acme/widgets"); + expect(alert?.repoFullName).toBe("acme/widgets"); + }); +}); diff --git a/test/unit/miner-governor-kill-switch.test.ts b/test/unit/miner-governor-kill-switch.test.ts index 412defcbf8..02a441f7bf 100644 --- a/test/unit/miner-governor-kill-switch.test.ts +++ b/test/unit/miner-governor-kill-switch.test.ts @@ -7,8 +7,37 @@ vi.mock("@loopover/engine", async () => { return import("../../packages/loopover-engine/src/index"); }); -import { checkMinerKillSwitch, recordMinerKillSwitchTransition } from "../../packages/loopover-miner/lib/governor-kill-switch.js"; +import { + checkMinerKillSwitch, + notifyMinerKillSwitchPagerDuty, + recordMinerKillSwitchTransition, +} from "../../packages/loopover-miner/lib/governor-kill-switch.js"; import { initGovernorLedger } from "../../packages/loopover-miner/lib/governor-ledger.js"; +import type { MinerKillSwitchPagerDutyAlert } from "../../packages/loopover-engine/src/index"; + +const VALID_ROUTING_KEY = "a".repeat(32); + +function stubFetch(status = 202): Array<{ url: string; body: Record }> { + const calls: Array<{ url: string; body: Record }> = []; + vi.stubGlobal("fetch", async (url: RequestInfo | URL, init?: RequestInit) => { + calls.push({ url: String(url), body: init?.body ? (JSON.parse(String(init.body)) as Record) : {} }); + return new Response(null, { status }); + }); + return calls; +} + +function pagerDutyAlert(over: Partial = {}): MinerKillSwitchPagerDutyAlert { + return { + repoFullName: "acme/widgets", + scope: "repo", + actionClass: "open_pr", + summary: "AMS miner kill-switch tripped (repo) — open_pr halted for acme/widgets", + severity: "critical", + dedupKey: "miner_kill_switch_tripped:repo:acme/widgets", + customDetails: { scope: "repo", previousScope: "none", repoFullName: "acme/widgets", actionClass: "open_pr" }, + ...over, + }; +} const roots: string[] = []; const ledgers: Array<{ close(): void }> = []; @@ -16,6 +45,7 @@ const ledgers: Array<{ close(): void }> = []; afterEach(() => { for (const ledger of ledgers.splice(0)) ledger.close(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); + vi.unstubAllGlobals(); }); describe("checkMinerKillSwitch (#2341)", () => { @@ -133,4 +163,220 @@ describe("recordMinerKillSwitchTransition (#2341)", () => { else process.env.LOOPOVER_MINER_GOVERNOR_LEDGER_DB = previousDbPath; } }); + + it("pages PagerDuty on a TRIP transition, after the ledger row is appended (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-trip-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const calls: MinerKillSwitchPagerDutyAlert[] = []; + const notify = vi.fn((alert: MinerKillSwitchPagerDutyAlert) => { + // The ledger row must already be visible by the time notify fires. + expect(ledger.readGovernorEvents({ repoFullName: "acme/widgets" })).toHaveLength(1); + calls.push(alert); + }); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event), notify, env: {} }, + ); + + expect(tripped?.decision).toBe("tripped"); + expect(notify).toHaveBeenCalledTimes(1); + expect(calls[0]).toMatchObject({ repoFullName: "acme/widgets", scope: "repo", dedupKey: "miner_kill_switch_tripped:repo:acme/widgets" }); + }); + + it("does NOT page PagerDuty on a resume transition (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-resume-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const notify = vi.fn(); + + const resumed = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "repo", scope: "none" }, + { append: (event) => ledger.appendGovernorEvent(event), notify, env: {} }, + ); + + expect(resumed?.decision).toBe("resumed"); + expect(notify).not.toHaveBeenCalled(); + }); + + it("a synchronously-throwing notify hook is swallowed and never blocks the returned ledger entry (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-sync-throw-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const notify = vi.fn(() => { + throw new Error("pagerduty transport down"); + }); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event), notify, env: {} }, + ); + + expect(tripped?.decision).toBe("tripped"); + expect(notify).toHaveBeenCalledTimes(1); + }); + + it("an asynchronously-rejecting notify hook is caught and never surfaces as unhandled (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-async-reject-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const notify = vi.fn(async () => { + throw new Error("pagerduty http 500"); + }); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event), notify, env: {} }, + ); + + expect(tripped?.decision).toBe("tripped"); + // Let the fire-and-forget rejection's own .catch handler run so it never surfaces as unhandled. + await Promise.resolve(); + await Promise.resolve(); + }); + + it("a sync notify hook returning void (not a promise) is accepted without calling .catch on it (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-sync-void-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const notify = vi.fn(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event), notify, env: {} }, + ); + + expect(tripped?.decision).toBe("tripped"); + expect(notify).toHaveBeenCalledTimes(1); + }); + + it("defaults to the real notifyMinerKillSwitchPagerDuty + process.env when no notify/env override is passed (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-default-notify-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const calls = stubFetch(); + + // LOOPOVER_ENABLE_PAGERDUTY is unset in the test environment, so the real default notify path resolves to a + // no-op -- this exercises the "no notify/env option passed" default-parameter branches themselves; the live + // network call's own guard branches are covered in the notifyMinerKillSwitchPagerDuty describe block below. + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event) }, + ); + + expect(tripped?.decision).toBe("tripped"); + expect(calls).toHaveLength(0); + }); +}); + +describe("notifyMinerKillSwitchPagerDuty (#7666)", () => { + it("no-op when LOOPOVER_ENABLE_PAGERDUTY is not truthy", async () => { + const calls = stubFetch(); + await notifyMinerKillSwitchPagerDuty(pagerDutyAlert(), {}); + expect(calls).toHaveLength(0); + }); + + it("no-op when the flag is on but no routing key resolves", async () => { + const calls = stubFetch(); + await notifyMinerKillSwitchPagerDuty(pagerDutyAlert(), { LOOPOVER_ENABLE_PAGERDUTY: "true" }); + expect(calls).toHaveLength(0); + }); + + it("no-op when the routing key is present but malformed", async () => { + const calls = stubFetch(); + await notifyMinerKillSwitchPagerDuty(pagerDutyAlert(), { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: "not-hex" }); + expect(calls).toHaveLength(0); + }); + + it("no-op when the routing key is present but blank/whitespace-only (envString's trim-to-empty branch)", async () => { + const calls = stubFetch(); + await notifyMinerKillSwitchPagerDuty(pagerDutyAlert(), { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: " " }); + expect(calls).toHaveLength(0); + }); + + it("fires the Events API v2 enqueue call when enabled and configured, for a repo-scoped alert", async () => { + const calls = stubFetch(202); + const alert = pagerDutyAlert(); + + await notifyMinerKillSwitchPagerDuty(alert, { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_ROUTING_KEY }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe("https://events.pagerduty.com/v2/enqueue"); + const body = calls[0]?.body as { routing_key: string; event_action: string; dedup_key: string; payload: { severity: string; component: string; source: string } }; + expect(body.routing_key).toBe(VALID_ROUTING_KEY); + expect(body.event_action).toBe("trigger"); + expect(body.dedup_key).toBe(alert.dedupKey); + expect(body.payload.severity).toBe("critical"); + expect(body.payload.component).toBe("acme/widgets"); + expect(body.payload.source).toBe("loopover-miner"); + }); + + it("a global-scope alert (no repoFullName) reports 'global' as the payload component", async () => { + const calls = stubFetch(202); + const alert = pagerDutyAlert({ repoFullName: null, scope: "global", dedupKey: "miner_kill_switch_tripped:global:global" }); + + await notifyMinerKillSwitchPagerDuty(alert, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: VALID_ROUTING_KEY }); + + const body = calls[0]?.body as { payload: { component: string } }; + expect(body.payload.component).toBe("global"); + }); + + it("a non-ok response is warned but never throws", async () => { + stubFetch(500); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await notifyMinerKillSwitchPagerDuty(pagerDutyAlert(), { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_ROUTING_KEY }); + + expect(warnSpy).toHaveBeenCalledTimes(1); + warnSpy.mockRestore(); + }); + + it("a thrown fetch error is caught and warned, never throws", async () => { + vi.stubGlobal("fetch", async () => { + throw new Error("network down"); + }); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await expect(notifyMinerKillSwitchPagerDuty(pagerDutyAlert(), { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_ROUTING_KEY })).resolves.toBeUndefined(); + + expect(warnSpy).toHaveBeenCalledTimes(1); + warnSpy.mockRestore(); + }); + + it("a thrown non-Error value is coerced via String(), not read as .message", async () => { + vi.stubGlobal("fetch", async () => { + // eslint-disable-next-line @typescript-eslint/no-throw-literal -- deliberately non-Error, to exercise + // warnMinerKillSwitchPagerDutyFailed's String(error) coercion branch (not every thrown value is an Error). + throw "connection refused"; + }); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await expect(notifyMinerKillSwitchPagerDuty(pagerDutyAlert(), { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_ROUTING_KEY })).resolves.toBeUndefined(); + + expect(warnSpy).toHaveBeenCalledTimes(1); + const [warnedJson] = warnSpy.mock.calls[0] ?? []; + expect(String(warnedJson)).toContain("connection refused"); + warnSpy.mockRestore(); + }); + + it("falls back to process.env when no env override is passed", async () => { + const hadFlag = Object.prototype.hasOwnProperty.call(process.env, "LOOPOVER_ENABLE_PAGERDUTY"); + const previousFlag = process.env.LOOPOVER_ENABLE_PAGERDUTY; + delete process.env.LOOPOVER_ENABLE_PAGERDUTY; + const calls = stubFetch(); + + try { + await notifyMinerKillSwitchPagerDuty(pagerDutyAlert()); + expect(calls).toHaveLength(0); + } finally { + if (hadFlag) process.env.LOOPOVER_ENABLE_PAGERDUTY = previousFlag; + } + }); });