diff --git a/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx b/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx index 90a9bd9289..ac5f455ab5 100644 --- a/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx +++ b/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx @@ -52,6 +52,13 @@ Flag a misbehaving loop from any of: 3. Fleet observability — error-rate spikes, abnormal claim/submission patterns, elevated `consecutive_failures` / rejection-reason clustering, or an explicit customer/support escalation (see the rented-loop escalation path from #4806). +4. **Automated PagerDuty page** (#7666) — when `LOOPOVER_ENABLE_PAGERDUTY` is on and + `PAGERDUTY_ROUTING_KEY` is configured in the **miner** process, a kill-switch **trip** (engage) + fires an Events API v2 page using the same flag / routing-key / enqueue contract as ORB's + `notify-pagerduty` module. A resume does not page. The miner has no D1-backed severity floor or + cooldown (PagerDuty's own `dedup_key` still coalesces duplicate incidents). Treat an + `ams_kill_switch:*` incident as "operator already needed on the page" and start the 15-minute + response clock immediately. **Triage before acting:** decide one-repo vs fleet-wide. Prefer the narrowest scope that stops the harm so one bad tenant does not force an unnecessary global halt. diff --git a/packages/loopover-engine/src/governor/kill-switch.ts b/packages/loopover-engine/src/governor/kill-switch.ts index 1f49f119e9..f08af3da52 100644 --- a/packages/loopover-engine/src/governor/kill-switch.ts +++ b/packages/loopover-engine/src/governor/kill-switch.ts @@ -7,7 +7,9 @@ // // DETECTOR ONLY — no IO, no persistence. Composing this with the other pure calculators into one fail-closed // allow/deny verdict (and recording every CHECK, not just a transition) is the Governor chokepoint's job -// (#2340), which consults this module first in its "safest wins" precedence. +// (#2340), which consults this module first in its "safest wins" precedence. Paging on a trip (#7666) is also +// an IO concern: this module only builds the pure PagerDuty alert payload; the miner IO seam fires the +// Events API call (mirroring `src/services/notify-pagerduty.ts`'s contract — AMS trips have no hosted path). import type { GovernorLedgerEvent } from "../governor-ledger.js"; @@ -73,3 +75,43 @@ export function buildMinerKillSwitchTransitionGovernorLedgerEvent(input: { payload: { previousScope: input.previousScope, scope: input.scope }, }; } + +/** + * Pure page payload for a kill-switch TRIP (#7666). Returns `null` when the transition is not a trip + * (no-op same-scope, or a resume) — paging wakes humans for engage, not for clear. `repoFullName` falls + * back to `ams/fleet` for a global halt with no single-repo context so routing still resolves against the + * operator's global PagerDuty key. Consumers (miner IO seam) own the actual Events API call — there is no + * hosted AMS trip path today, so this module stays detector-only. + */ +export type MinerKillSwitchPagerDutyAlert = { + repoFullName: string; + summary: string; + severity: "critical"; + dedupKey: string; + customDetails: { + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; + reason: string; + }; +}; + +export function buildMinerKillSwitchPagerDutyAlert(input: { + repoFullName?: string | null | undefined; + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; +}): MinerKillSwitchPagerDutyAlert | null { + if (input.previousScope === input.scope) return null; + if (!isMinerKillSwitchActive(input.scope)) return null; + const repoFullName = (input.repoFullName ?? "").trim() || "ams/fleet"; + const reason = `${input.scope}_kill_switch_engaged`; + return { + repoFullName, + summary: + input.scope === "global" + ? `AMS miner kill-switch engaged (global / fleet-wide)` + : `AMS miner kill-switch engaged (repo) for ${repoFullName}`, + severity: "critical", + dedupKey: `ams_kill_switch:${input.scope}:${repoFullName.toLowerCase()}`, + customDetails: { previousScope: input.previousScope, scope: input.scope, reason }, + }; +} diff --git a/packages/loopover-engine/test/kill-switch.test.ts b/packages/loopover-engine/test/kill-switch.test.ts index b06e5a5c15..4c563a2718 100644 --- a/packages/loopover-engine/test/kill-switch.test.ts +++ b/packages/loopover-engine/test/kill-switch.test.ts @@ -3,6 +3,7 @@ import { test } from "node:test"; import { MINER_KILL_SWITCH_ENV_VAR, + buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, @@ -14,6 +15,7 @@ test("barrel: the public entrypoint re-exports the kill-switch primitive (#2341) assert.equal(typeof resolveMinerKillSwitch, "function"); assert.equal(typeof isMinerKillSwitchActive, "function"); assert.equal(typeof buildMinerKillSwitchTransitionGovernorLedgerEvent, "function"); + assert.equal(typeof buildMinerKillSwitchPagerDutyAlert, "function"); assert.equal(MINER_KILL_SWITCH_ENV_VAR, "LOOPOVER_MINER_KILL_SWITCH"); }); @@ -105,3 +107,46 @@ test("buildMinerKillSwitchTransitionGovernorLedgerEvent: clearing the switch rec payload: { previousScope: "global", scope: "none" }, }); }); + +test("buildMinerKillSwitchPagerDutyAlert: trip builds a critical page payload (#7666)", () => { + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: "acme/widgets", + previousScope: "none", + scope: "repo", + }); + assert.deepEqual(alert, { + repoFullName: "acme/widgets", + summary: "AMS miner kill-switch engaged (repo) for acme/widgets", + severity: "critical", + dedupKey: "ams_kill_switch:repo:acme/widgets", + customDetails: { previousScope: "none", scope: "repo", reason: "repo_kill_switch_engaged" }, + }); +}); + +test("buildMinerKillSwitchPagerDutyAlert: global trip without a repo uses ams/fleet (#7666)", () => { + const alert = buildMinerKillSwitchPagerDutyAlert({ + previousScope: "none", + scope: "global", + }); + assert.equal(alert?.repoFullName, "ams/fleet"); + assert.equal(alert?.dedupKey, "ams_kill_switch:global:ams/fleet"); + assert.match(alert?.summary ?? "", /fleet-wide/); + + const blankRepo = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: " ", + previousScope: "none", + scope: "global", + }); + assert.equal(blankRepo?.repoFullName, "ams/fleet"); +}); + +test("buildMinerKillSwitchPagerDutyAlert: resume / same-scope are silent (#7666)", () => { + assert.equal( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "none" }), + null, + ); + assert.equal( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "repo" }), + null, + ); +}); diff --git a/packages/loopover-miner/lib/governor-kill-switch.d.ts b/packages/loopover-miner/lib/governor-kill-switch.d.ts index 191fa6e423..7f7a100962 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.d.ts +++ b/packages/loopover-miner/lib/governor-kill-switch.d.ts @@ -1,3 +1,4 @@ +import { type MinerKillSwitchPagerDutyAlert } from "@loopover/engine"; import type { MinerKillSwitchScope } from "@loopover/engine"; import type { AppendGovernorEventInput, GovernorLedgerEntry } from "./governor-ledger.js"; export type CheckMinerKillSwitchInput = { @@ -19,11 +20,21 @@ export type RecordMinerKillSwitchTransitionInput = { previousScope: MinerKillSwitchScope; scope: MinerKillSwitchScope; }; +export type NotifyMinerKillSwitchTrip = (alert: MinerKillSwitchPagerDutyAlert, env: Record) => void | Promise; +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export declare function notifyMinerKillSwitchPagerDuty(alert: MinerKillSwitchPagerDutyAlert, env?: Record): Promise; /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export declare function recordMinerKillSwitchTransition(input: RecordMinerKillSwitchTransitionInput, options?: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry; + notify?: NotifyMinerKillSwitchTrip; + env?: Record; }): GovernorLedgerEntry | null; diff --git a/packages/loopover-miner/lib/governor-kill-switch.js b/packages/loopover-miner/lib/governor-kill-switch.js index 3c479144f3..bb1e4e2980 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.js +++ b/packages/loopover-miner/lib/governor-kill-switch.js @@ -2,7 +2,14 @@ // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence. -import { buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, resolveMinerKillSwitch, } from "@loopover/engine"; +// +// #7666: a TRIP also pages via the same PagerDuty Events API v2 contract ORB uses in +// `src/services/notify-pagerduty.ts` (LOOPOVER_ENABLE_PAGERDUTY + PAGERDUTY_ROUTING_KEY + enqueue URL + +// dedup_key). AMS trips only exist in this miner process (no hosted trip call site / no Worker Env), so +// the page lives here rather than calling `triggerPagerDutyIncident` directly. Resume stays silent — +// clearing a halt must not wake anyone. Best-effort and never throws: a paging failure must never block +// the ledger write or the mid-attempt abandon that depends on it. +import { buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, resolveMinerKillSwitch, } from "@loopover/engine"; import { appendGovernorEvent } from "./governor-ledger.js"; /** * Resolve the current kill-switch scope for a repo from process env plus a per-repo paused flag (typically @@ -14,16 +21,93 @@ export function checkMinerKillSwitch(input = {}) { const scope = resolveMinerKillSwitch({ global, repoPaused: input.repoPaused }); return { scope, active: isMinerKillSwitchActive(scope) }; } +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; +const TRUTHY_ENV = /^(1|true|yes|on)$/i; +function envString(env, name) { + const value = env[name]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} +function pagerDutyFailMessage(error) { + // Prefer Error.message when present; otherwise coerce. Single helper so both sync and async + // failure paths share one branch surface for Codecov patch. + return (error instanceof Error ? error.message : String(error)).slice(0, 200); +} +function warnKillSwitchPagerDutyFailed(repo, error) { + console.warn(JSON.stringify({ event: "kill_switch_pagerduty_failed", repo, message: pagerDutyFailMessage(error) })); +} +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export async function notifyMinerKillSwitchPagerDuty(alert, env = process.env) { + if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) + return; + const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY"); + if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) + return; + try { + const response = await fetch(PAGERDUTY_EVENTS_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: routingKey, + event_action: "trigger", + dedup_key: alert.dedupKey, + payload: { + summary: alert.summary.slice(0, 1024), + source: "loopover-miner", + severity: alert.severity, + timestamp: new Date().toISOString(), + component: alert.repoFullName, + custom_details: alert.customDetails, + }, + }), + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) { + console.warn(JSON.stringify({ + event: "kill_switch_pagerduty_failed", + repo: alert.repoFullName, + status: response.status, + })); + } + } + catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } +} /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export function recordMinerKillSwitchTransition(input, options = {}) { const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input); if (!event) return null; const append = options.append ?? appendGovernorEvent; - return append(event); + const recorded = append(event); + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: input.repoFullName, + previousScope: input.previousScope, + scope: input.scope, + }); + if (alert) { + const notify = options.notify ?? notifyMinerKillSwitchPagerDuty; + const env = options.env ?? process.env; + try { + // Promise.resolve wraps sync returns so both sync throws and async rejects share one failure path. + void Promise.resolve(notify(alert, env)).catch((error) => { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + }); + } + catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } + } + return recorded; } -//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ292ZXJub3Ita2lsbC1zd2l0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJnb3Zlcm5vci1raWxsLXN3aXRjaC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSw4R0FBOEc7QUFDOUcsd0dBQXdHO0FBQ3hHLDJHQUEyRztBQUMzRyx1R0FBdUc7QUFFdkcsT0FBTyxFQUNMLGlEQUFpRCxFQUNqRCx1QkFBdUIsRUFDdkIsdUJBQXVCLEVBQ3ZCLHNCQUFzQixHQUN2QixNQUFNLGtCQUFrQixDQUFDO0FBRTFCLE9BQU8sRUFBRSxtQkFBbUIsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBYTNEOzs7R0FHRztBQUNILE1BQU0sVUFBVSxvQkFBb0IsQ0FBQyxRQUFtQyxFQUFFO0lBQ3hFLE1BQU0sR0FBRyxHQUFHLEtBQUssQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQztJQUNyQyxNQUFNLE1BQU0sR0FBRyx1QkFBdUIsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUM1QyxNQUFNLEtBQUssR0FBRyxzQkFBc0IsQ0FBQyxFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsS0FBSyxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUM7SUFDL0UsT0FBTyxFQUFFLEtBQUssRUFBRSxNQUFNLEVBQUUsdUJBQXVCLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztBQUMzRCxDQUFDO0FBU0Q7Ozs7R0FJRztBQUNILE1BQU0sVUFBVSwrQkFBK0IsQ0FDN0MsS0FBMkMsRUFDM0MsVUFBaUYsRUFBRTtJQUVuRixNQUFNLEtBQUssR0FBRyxpREFBaUQsQ0FBQyxLQUFLLENBQUMsQ0FBQztJQUN2RSxJQUFJLENBQUMsS0FBSztRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ3hCLE1BQU0sTUFBTSxHQUFHLE9BQU8sQ0FBQyxNQUFNLElBQUksbUJBQW1CLENBQUM7SUFDckQsT0FBTyxNQUFNLENBQUMsS0FBaUMsQ0FBQyxDQUFDO0FBQ25ELENBQUMifQ== \ No newline at end of file +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ292ZXJub3Ita2lsbC1zd2l0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJnb3Zlcm5vci1raWxsLXN3aXRjaC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSw4R0FBOEc7QUFDOUcsd0dBQXdHO0FBQ3hHLDJHQUEyRztBQUMzRyx1R0FBdUc7QUFDdkcsRUFBRTtBQUNGLHFGQUFxRjtBQUNyRix3R0FBd0c7QUFDeEcsd0dBQXdHO0FBQ3hHLHFHQUFxRztBQUNyRyx3R0FBd0c7QUFDeEcsa0VBQWtFO0FBRWxFLE9BQU8sRUFDTCxrQ0FBa0MsRUFDbEMsaURBQWlELEVBQ2pELHVCQUF1QixFQUN2Qix1QkFBdUIsRUFDdkIsc0JBQXNCLEdBRXZCLE1BQU0sa0JBQWtCLENBQUM7QUFFMUIsT0FBTyxFQUFFLG1CQUFtQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFhM0Q7OztHQUdHO0FBQ0gsTUFBTSxVQUFVLG9CQUFvQixDQUFDLFFBQW1DLEVBQUU7SUFDeEUsTUFBTSxHQUFHLEdBQUcsS0FBSyxDQUFDLEdBQUcsSUFBSSxPQUFPLENBQUMsR0FBRyxDQUFDO0lBQ3JDLE1BQU0sTUFBTSxHQUFHLHVCQUF1QixDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQzVDLE1BQU0sS0FBSyxHQUFHLHNCQUFzQixDQUFDLEVBQUUsTUFBTSxFQUFFLFVBQVUsRUFBRSxLQUFLLENBQUMsVUFBVSxFQUFFLENBQUMsQ0FBQztJQUMvRSxPQUFPLEVBQUUsS0FBSyxFQUFFLE1BQU0sRUFBRSx1QkFBdUIsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDO0FBQzNELENBQUM7QUFjRCxNQUFNLG9CQUFvQixHQUFHLHlDQUF5QyxDQUFDO0FBQ3ZFLE1BQU0sY0FBYyxHQUFHLGlCQUFpQixDQUFDO0FBQ3pDLE1BQU0sVUFBVSxHQUFHLG9CQUFvQixDQUFDO0FBRXhDLFNBQVMsU0FBUyxDQUFDLEdBQXVDLEVBQUUsSUFBWTtJQUN0RSxNQUFNLEtBQUssR0FBRyxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDeEIsT0FBTyxPQUFPLEtBQUssS0FBSyxRQUFRLElBQUksS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUFDLENBQUMsU0FBUyxDQUFDO0FBQ3pGLENBQUM7QUFFRCxTQUFTLG9CQUFvQixDQUFDLEtBQWM7SUFDMUMsNEZBQTRGO0lBQzVGLDREQUE0RDtJQUM1RCxPQUFPLENBQUMsS0FBSyxZQUFZLEtBQUssQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsRUFBRSxHQUFHLENBQUMsQ0FBQztBQUNoRixDQUFDO0FBRUQsU0FBUyw2QkFBNkIsQ0FBQyxJQUFZLEVBQUUsS0FBYztJQUNqRSxPQUFPLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsRUFBRSxLQUFLLEVBQUUsOEJBQThCLEVBQUUsSUFBSSxFQUFFLE9BQU8sRUFBRSxvQkFBb0IsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQztBQUN0SCxDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUsOEJBQThCLENBQ2xELEtBQW9DLEVBQ3BDLE1BQTBDLE9BQU8sQ0FBQyxHQUFHO0lBRXJELElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLENBQUMsR0FBRyxDQUFDLHlCQUF5QixJQUFJLEVBQUUsQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDO1FBQUUsT0FBTztJQUMzRSxNQUFNLFVBQVUsR0FBRyxTQUFTLENBQUMsR0FBRyxFQUFFLHVCQUF1QixDQUFDLENBQUM7SUFDM0QsSUFBSSxDQUFDLFVBQVUsSUFBSSxDQUFDLGNBQWMsQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDO1FBQUUsT0FBTztJQUU1RCxJQUFJLENBQUM7UUFDSCxNQUFNLFFBQVEsR0FBRyxNQUFNLEtBQUssQ0FBQyxvQkFBb0IsRUFBRTtZQUNqRCxNQUFNLEVBQUUsTUFBTTtZQUNkLE9BQU8sRUFBRSxFQUFFLGNBQWMsRUFBRSxrQkFBa0IsRUFBRTtZQUMvQyxJQUFJLEVBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQztnQkFDbkIsV0FBVyxFQUFFLFVBQVU7Z0JBQ3ZCLFlBQVksRUFBRSxTQUFTO2dCQUN2QixTQUFTLEVBQUUsS0FBSyxDQUFDLFFBQVE7Z0JBQ3pCLE9BQU8sRUFBRTtvQkFDUCxPQUFPLEVBQUUsS0FBSyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLElBQUksQ0FBQztvQkFDckMsTUFBTSxFQUFFLGdCQUFnQjtvQkFDeEIsUUFBUSxFQUFFLEtBQUssQ0FBQyxRQUFRO29CQUN4QixTQUFTLEVBQUUsSUFBSSxJQUFJLEVBQUUsQ0FBQyxXQUFXLEVBQUU7b0JBQ25DLFNBQVMsRUFBRSxLQUFLLENBQUMsWUFBWTtvQkFDN0IsY0FBYyxFQUFFLEtBQUssQ0FBQyxhQUFhO2lCQUNwQzthQUNGLENBQUM7WUFDRixNQUFNLEVBQUUsV0FBVyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUM7U0FDbEMsQ0FBQyxDQUFDO1FBQ0gsSUFBSSxDQUFDLFFBQVEsQ0FBQyxFQUFFLEVBQUUsQ0FBQztZQUNqQixPQUFPLENBQUMsSUFBSSxDQUNWLElBQUksQ0FBQyxTQUFTLENBQUM7Z0JBQ2IsS0FBSyxFQUFFLDhCQUE4QjtnQkFDckMsSUFBSSxFQUFFLEtBQUssQ0FBQyxZQUFZO2dCQUN4QixNQUFNLEVBQUUsUUFBUSxDQUFDLE1BQU07YUFDeEIsQ0FBQyxDQUNILENBQUM7UUFDSixDQUFDO0lBQ0gsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDZiw2QkFBNkIsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLEtBQUssQ0FBQyxDQUFDO0lBQzNELENBQUM7QUFDSCxDQUFDO0FBRUQ7Ozs7O0dBS0c7QUFDSCxNQUFNLFVBQVUsK0JBQStCLENBQzdDLEtBQTJDLEVBQzNDLFVBSUksRUFBRTtJQUVOLE1BQU0sS0FBSyxHQUFHLGlEQUFpRCxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ3ZFLElBQUksQ0FBQyxLQUFLO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDeEIsTUFBTSxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sSUFBSSxtQkFBbUIsQ0FBQztJQUNyRCxNQUFNLFFBQVEsR0FBRyxNQUFNLENBQUMsS0FBaUMsQ0FBQyxDQUFDO0lBRTNELE1BQU0sS0FBSyxHQUFHLGtDQUFrQyxDQUFDO1FBQy9DLFlBQVksRUFBRSxLQUFLLENBQUMsWUFBWTtRQUNoQyxhQUFhLEVBQUUsS0FBSyxDQUFDLGFBQWE7UUFDbEMsS0FBSyxFQUFFLEtBQUssQ0FBQyxLQUFLO0tBQ25CLENBQUMsQ0FBQztJQUNILElBQUksS0FBSyxFQUFFLENBQUM7UUFDVixNQUFNLE1BQU0sR0FBRyxPQUFPLENBQUMsTUFBTSxJQUFJLDhCQUE4QixDQUFDO1FBQ2hFLE1BQU0sR0FBRyxHQUFHLE9BQU8sQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQztRQUN2QyxJQUFJLENBQUM7WUFDSCxtR0FBbUc7WUFDbkcsS0FBSyxPQUFPLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLEVBQUUsR0FBRyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxLQUFjLEVBQUUsRUFBRTtnQkFDaEUsNkJBQTZCLENBQUMsS0FBSyxDQUFDLFlBQVksRUFBRSxLQUFLLENBQUMsQ0FBQztZQUMzRCxDQUFDLENBQUMsQ0FBQztRQUNMLENBQUM7UUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1lBQ2YsNkJBQTZCLENBQUMsS0FBSyxDQUFDLFlBQVksRUFBRSxLQUFLLENBQUMsQ0FBQztRQUMzRCxDQUFDO0lBQ0gsQ0FBQztJQUVELE9BQU8sUUFBUSxDQUFDO0FBQ2xCLENBQUMifQ== \ No newline at end of file diff --git a/packages/loopover-miner/lib/governor-kill-switch.ts b/packages/loopover-miner/lib/governor-kill-switch.ts index 71a4de6773..e580ecfd58 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.ts +++ b/packages/loopover-miner/lib/governor-kill-switch.ts @@ -2,12 +2,21 @@ // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence. +// +// #7666: a TRIP also pages via the same PagerDuty Events API v2 contract ORB uses in +// `src/services/notify-pagerduty.ts` (LOOPOVER_ENABLE_PAGERDUTY + PAGERDUTY_ROUTING_KEY + enqueue URL + +// dedup_key). AMS trips only exist in this miner process (no hosted trip call site / no Worker Env), so +// the page lives here rather than calling `triggerPagerDutyIncident` directly. Resume stays silent — +// clearing a halt must not wake anyone. Best-effort and never throws: a paging failure must never block +// the ledger write or the mid-attempt abandon that depends on it. import { + buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, resolveMinerKillSwitch, + type MinerKillSwitchPagerDutyAlert, } from "@loopover/engine"; import type { MinerKillSwitchScope } from "@loopover/engine"; import { appendGovernorEvent } from "./governor-ledger.js"; @@ -41,17 +50,112 @@ export type RecordMinerKillSwitchTransitionInput = { scope: MinerKillSwitchScope; }; +export type NotifyMinerKillSwitchTrip = ( + alert: MinerKillSwitchPagerDutyAlert, + env: Record, +) => void | Promise; + +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; +const TRUTHY_ENV = /^(1|true|yes|on)$/i; + +function envString(env: Record, name: string): string | undefined { + const value = env[name]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} + +function pagerDutyFailMessage(error: unknown): string { + // Prefer Error.message when present; otherwise coerce. Single helper so both sync and async + // failure paths share one branch surface for Codecov patch. + return (error instanceof Error ? error.message : String(error)).slice(0, 200); +} + +function warnKillSwitchPagerDutyFailed(repo: string, error: unknown): void { + console.warn(JSON.stringify({ event: "kill_switch_pagerduty_failed", repo, message: pagerDutyFailMessage(error) })); +} + +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export async function notifyMinerKillSwitchPagerDuty( + alert: MinerKillSwitchPagerDutyAlert, + env: Record = process.env, +): Promise { + if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) return; + const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY"); + if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) return; + + try { + const response = await fetch(PAGERDUTY_EVENTS_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: routingKey, + event_action: "trigger", + dedup_key: alert.dedupKey, + payload: { + summary: alert.summary.slice(0, 1024), + source: "loopover-miner", + severity: alert.severity, + timestamp: new Date().toISOString(), + component: alert.repoFullName, + custom_details: alert.customDetails, + }, + }), + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) { + console.warn( + JSON.stringify({ + event: "kill_switch_pagerduty_failed", + repo: alert.repoFullName, + status: response.status, + }), + ); + } + } catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } +} + /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export function recordMinerKillSwitchTransition( input: RecordMinerKillSwitchTransitionInput, - options: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry } = {}, + options: { + append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry; + notify?: NotifyMinerKillSwitchTrip; + env?: Record; + } = {}, ): GovernorLedgerEntry | null { const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input); if (!event) return null; const append = options.append ?? appendGovernorEvent; - return append(event as AppendGovernorEventInput); + const recorded = append(event as AppendGovernorEventInput); + + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: input.repoFullName, + previousScope: input.previousScope, + scope: input.scope, + }); + if (alert) { + const notify = options.notify ?? notifyMinerKillSwitchPagerDuty; + const env = options.env ?? process.env; + try { + // Promise.resolve wraps sync returns so both sync throws and async rejects share one failure path. + void Promise.resolve(notify(alert, env)).catch((error: unknown) => { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + }); + } catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } + } + + return recorded; } diff --git a/src/services/notify-pagerduty.ts b/src/services/notify-pagerduty.ts index 073aeaf4a0..58b7555ebd 100644 --- a/src/services/notify-pagerduty.ts +++ b/src/services/notify-pagerduty.ts @@ -218,3 +218,9 @@ export async function triggerPagerDutyIncident( await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "error", message.slice(0, 280)); } } + +// AMS miner kill-switch trips (#7666) are recorded only in the miner process +// (`packages/loopover-miner/lib/governor-kill-switch.ts` → `recordMinerKillSwitchTransition`). That path +// mirrors this module's Events API v2 contract (same flag + routing key + enqueue URL + dedup_key shape) +// because the miner has no Worker Env/D1 for severity-floor / cooldown audit. Do not add a hosted wrapper +// here without a real hosted AMS trip call site — there isn't one today. diff --git a/test/unit/kill-switch-incident-runbook.test.ts b/test/unit/kill-switch-incident-runbook.test.ts index 582fc5e0fd..299bbb9c67 100644 --- a/test/unit/kill-switch-incident-runbook.test.ts +++ b/test/unit/kill-switch-incident-runbook.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest"; import { MINER_KILL_SWITCH_ENV_VAR, + buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, resolveMinerKillSwitch, } from "../../packages/loopover-engine/src/governor/kill-switch"; @@ -36,6 +37,8 @@ describe("kill-switch incident runbook (#4809)", () => { expect(runbook).toContain("15 minutes"); expect(runbook).toContain("2 minutes"); expect(runbook).toContain("#7180"); + expect(runbook).toContain("LOOPOVER_ENABLE_PAGERDUTY"); + expect(runbook).toContain("ams_kill_switch"); expect(resolveMinerKillSwitch({ global: true, repoPaused: true })).toBe("global"); expect(resolveMinerKillSwitch({ global: false, repoPaused: true })).toBe("repo"); @@ -78,3 +81,39 @@ describe("kill-switch incident runbook (#4809)", () => { expect(runbook).toMatch(/do not reach for those commands/i); }); }); + +describe("buildMinerKillSwitchPagerDutyAlert under vitest (#7666)", () => { + it("covers trip / resume / same-scope / fleet / blank-repo branches", () => { + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "none", scope: "repo" }), + ).toMatchObject({ + repoFullName: "acme/widgets", + severity: "critical", + dedupKey: "ams_kill_switch:repo:acme/widgets", + }); + + expect( + buildMinerKillSwitchPagerDutyAlert({ previousScope: "none", scope: "global" }), + ).toMatchObject({ + repoFullName: "ams/fleet", + dedupKey: "ams_kill_switch:global:ams/fleet", + summary: expect.stringContaining("fleet-wide"), + }); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: null, previousScope: "none", scope: "global" }), + ).toMatchObject({ repoFullName: "ams/fleet" }); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: " ", previousScope: "none", scope: "global" }), + ).toMatchObject({ repoFullName: "ams/fleet" }); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "none" }), + ).toBeNull(); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "repo" }), + ).toBeNull(); + }); +}); diff --git a/test/unit/miner-governor-kill-switch.test.ts b/test/unit/miner-governor-kill-switch.test.ts index 412defcbf8..8d87a8aa12 100644 --- a/test/unit/miner-governor-kill-switch.test.ts +++ b/test/unit/miner-governor-kill-switch.test.ts @@ -7,7 +7,7 @@ vi.mock("@loopover/engine", async () => { return import("../../packages/loopover-engine/src/index"); }); -import { checkMinerKillSwitch, recordMinerKillSwitchTransition } from "../../packages/loopover-miner/lib/governor-kill-switch.js"; +import { checkMinerKillSwitch, notifyMinerKillSwitchPagerDuty, recordMinerKillSwitchTransition } from "../../packages/loopover-miner/lib/governor-kill-switch.js"; import { initGovernorLedger } from "../../packages/loopover-miner/lib/governor-ledger.js"; const roots: string[] = []; @@ -119,7 +119,7 @@ describe("recordMinerKillSwitchTransition (#2341)", () => { actionClass: "open_pr", previousScope: "none", scope: "repo", - }); + }, { notify: () => undefined }); expect(tripped?.decision).toBe("tripped"); closeDefaultGovernorLedger(); @@ -133,4 +133,289 @@ describe("recordMinerKillSwitchTransition (#2341)", () => { else process.env.LOOPOVER_MINER_GOVERNOR_LEDGER_DB = previousDbPath; } }); + + it("pages on trip via the injectable notify hook and stays silent on resume (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const notify = vi.fn(); + + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event), notify }, + ); + expect(notify).toHaveBeenCalledTimes(1); + expect(notify.mock.calls[0]?.[0]).toMatchObject({ + repoFullName: "acme/widgets", + severity: "critical", + dedupKey: "ams_kill_switch:repo:acme/widgets", + }); + + notify.mockClear(); + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "repo", scope: "none" }, + { append: (event) => ledger.appendGovernorEvent(event), notify }, + ); + expect(notify).not.toHaveBeenCalled(); + }); + + it("swallows a rejected notify promise without failing the ledger write (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-reject-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: async () => { + throw new Error("async pagerduty down"); + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + await vi.waitFor(() => { + expect(warn).toHaveBeenCalledWith(expect.stringContaining("kill_switch_pagerduty_failed")); + }); + warn.mockRestore(); + }); + + it("a sync void notify is accepted without treating it as a rejected promise (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-sync-void-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + let called = false; + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + called = true; + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(called).toBe(true); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it("swallows a sync Error throw from notify (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-error-throw-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "global" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + throw new Error("pagerduty error fail"); + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("pagerduty error fail")); + warn.mockRestore(); + }); + + it("swallows a sync non-Error throw from notify (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-string-throw-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "global" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + throw "pagerduty string fail"; + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("pagerduty string fail")); + warn.mockRestore(); + }); + + it("swallows a non-Error rejected notify promise (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-string-reject-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: async () => { + throw "async string fail"; + }, + }, + ); + await vi.waitFor(() => { + expect(warn).toHaveBeenCalledWith(expect.stringContaining("async string fail")); + }); + warn.mockRestore(); + }); + + it("uses the default notify + process.env when overrides are omitted (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-default-notify-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + const previousFlag = process.env.LOOPOVER_ENABLE_PAGERDUTY; + const previousKey = process.env.PAGERDUTY_ROUTING_KEY; + process.env.LOOPOVER_ENABLE_PAGERDUTY = "1"; + process.env.PAGERDUTY_ROUTING_KEY = "a".repeat(32); + try { + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event) }, + ); + expect(tripped?.decision).toBe("tripped"); + await vi.waitFor(() => { + expect(calls.length).toBeGreaterThan(0); + }); + } finally { + if (previousFlag === undefined) delete process.env.LOOPOVER_ENABLE_PAGERDUTY; + else process.env.LOOPOVER_ENABLE_PAGERDUTY = previousFlag; + if (previousKey === undefined) delete process.env.PAGERDUTY_ROUTING_KEY; + else process.env.PAGERDUTY_ROUTING_KEY = previousKey; + vi.unstubAllGlobals(); + } + }); +}); + +describe("notifyMinerKillSwitchPagerDuty (#7666)", () => { + const VALID_KEY = "a".repeat(32); + const ALERT = { + repoFullName: "acme/widgets", + summary: "AMS miner kill-switch engaged (repo) for acme/widgets", + severity: "critical" as const, + dedupKey: "ams_kill_switch:repo:acme/widgets", + customDetails: { previousScope: "none" as const, scope: "repo" as const, reason: "repo_kill_switch_engaged" }, + }; + + it("no-ops when the PagerDuty flag is off or unset", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "0", PAGERDUTY_ROUTING_KEY: VALID_KEY }); + await notifyMinerKillSwitchPagerDuty(ALERT, { PAGERDUTY_ROUTING_KEY: VALID_KEY }); + expect(calls).toHaveLength(0); + vi.unstubAllGlobals(); + }); + + it("posts Events API v2 when enabled with a valid routing key", async () => { + const calls: Array<{ url: string; body: Record }> = []; + vi.stubGlobal("fetch", async (url: RequestInfo | URL, init?: RequestInit) => { + calls.push({ url: String(url), body: init?.body ? (JSON.parse(String(init.body)) as Record) : {} }); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: VALID_KEY }); + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe("https://events.pagerduty.com/v2/enqueue"); + expect(calls[0]?.body).toMatchObject({ + routing_key: VALID_KEY, + event_action: "trigger", + dedup_key: "ams_kill_switch:repo:acme/widgets", + payload: { source: "loopover-miner", severity: "critical", component: "acme/widgets" }, + }); + vi.unstubAllGlobals(); + }); + + it("no-ops when the routing key is missing, blank, invalid, or non-string", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1" }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: " " }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: "not-a-key" }); + await notifyMinerKillSwitchPagerDuty(ALERT, { + LOOPOVER_ENABLE_PAGERDUTY: "1", + PAGERDUTY_ROUTING_KEY: 42 as unknown as string, + }); + expect(calls).toHaveLength(0); + vi.unstubAllGlobals(); + }); + + it("warns but does not throw when PagerDuty returns a non-ok status", async () => { + vi.stubGlobal("fetch", async () => new Response(null, { status: 500 })); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "yes", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("kill_switch_pagerduty_failed")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("never throws when fetch rejects with a non-Error value", async () => { + vi.stubGlobal("fetch", async () => { + throw "network string fail"; + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("network string fail")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("never throws when fetch rejects with an Error", async () => { + vi.stubGlobal("fetch", async () => { + throw new Error("network down"); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("network down")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("defaults env to process.env when the second arg is omitted", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + const previousFlag = process.env.LOOPOVER_ENABLE_PAGERDUTY; + const previousKey = process.env.PAGERDUTY_ROUTING_KEY; + process.env.LOOPOVER_ENABLE_PAGERDUTY = "on"; + process.env.PAGERDUTY_ROUTING_KEY = VALID_KEY; + try { + await notifyMinerKillSwitchPagerDuty(ALERT); + expect(calls).toHaveLength(1); + } finally { + if (previousFlag === undefined) delete process.env.LOOPOVER_ENABLE_PAGERDUTY; + else process.env.LOOPOVER_ENABLE_PAGERDUTY = previousFlag; + if (previousKey === undefined) delete process.env.PAGERDUTY_ROUTING_KEY; + else process.env.PAGERDUTY_ROUTING_KEY = previousKey; + vi.unstubAllGlobals(); + } + }); });