diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 3d31fb9c87..68f89951dd 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -643,6 +643,16 @@ export { type LoopHealthTier, type LoopRunOutcome, } from "./loop-escalation.js"; +// The internal ops fleet view (#4808) over the escalation vocabulary re-exported above: it calls the same +// evaluateEscalation per loop rather than restating what "needs a human" means. +export { + buildActiveLoopFleetSummary, + LOOP_HEALTH_TIERS, + LOOP_RUN_STATUSES, + type ActiveLoopFacts, + type ActiveLoopFleetSummary, + type FleetLoopRow, +} from "./loop-fleet-summary.js"; export { buildMetadataRankInput, computeMetadataDupRisk, diff --git a/packages/loopover-engine/src/loop-fleet-summary.ts b/packages/loopover-engine/src/loop-fleet-summary.ts new file mode 100644 index 0000000000..90fccbacb7 --- /dev/null +++ b/packages/loopover-engine/src/loop-fleet-summary.ts @@ -0,0 +1,109 @@ +// Active rented-loop fleet summary (pure) — #4808, part of the Rent-a-Loop path #4778. +// +// Deterministic and side-effect-free: given every rented loop the internal ops team currently knows about, it +// produces the at-a-glance view an operator needs — how many loops are live, how they break down by run status +// and health tier, and which ones are misbehaving badly enough to need a human right now. That is exactly +// #4808's acceptance criterion ("an internal operator can see every currently active rented loop and its status +// at a glance") as a decision core: what a dashboard panel renders and what an alert rule fires on, computed +// once, the same way, for both. +// +// It reuses loop-escalation.ts's (#4806) already-merged vocabulary rather than restating it — the same +// LoopRunOutcome/LoopHealthTier a loop is already described by, and evaluateEscalation itself to decide whether +// a given loop needs attention. So the fleet view can never disagree with the per-loop escalation path about +// what "needs a human" means: there is one rule, called once per loop, not a second copy that drifts. +// +// It summarizes only: no dashboard, no alert delivery, no IO, no clock read. Wiring panels/alert rules into the +// self-host observability stack is the separate integration this issue is blocked on (#4793) — this core has no +// opinion about Grafana or Alertmanager, so it stays correct whatever renders it. + +import { evaluateEscalation, type EscalationDecision, type LoopEscalationInput, type LoopHealthTier, type LoopRunOutcome } from "./loop-escalation.js"; + +/** Every run status a loop can report, in the order an operator reads them (live work first). */ +export const LOOP_RUN_STATUSES: readonly LoopRunOutcome[] = ["running", "converged", "abandoned", "error"]; +/** Health tiers, worst-first — the order the summary surfaces them in. */ +export const LOOP_HEALTH_TIERS: readonly LoopHealthTier[] = ["critical", "degraded", "healthy"]; + +/** One rented loop as ops currently knows it. Mirrors LoopEscalationInput's signals so the same rule applies. */ +export type ActiveLoopFacts = LoopEscalationInput & { + loopId: string; + tenantId: string; +}; + +/** One row of the operator's view: the loop, plus the escalation decision computed for it. */ +export type FleetLoopRow = { + loopId: string; + tenantId: string; + runStatus: LoopRunOutcome; + /** Absent when nothing has computed a health tier for this loop yet — reported as "unknown", never guessed. */ + healthStatus: LoopHealthTier | "unknown"; + needsAttention: boolean; + escalation: EscalationDecision; +}; + +export type ActiveLoopFleetSummary = { + /** Loops still running — the "currently active" count an operator reads first. */ + activeCount: number; + /** Every loop handed in, however it ended. */ + totalCount: number; + /** Count per run status. Every status is always present (0 when none), so a panel never renders a hole. */ + byStatus: Record; + /** Count per health tier, plus `unknown` for loops with no tier computed yet. Always fully populated. */ + byHealth: Record; + /** Loops needing a human, worst-first — what an alert rule fires on. A subset of `loops`, never a copy that drifts. */ + needingAttention: FleetLoopRow[]; + /** Every loop, in a stable order. */ + loops: FleetLoopRow[]; +}; + +/** Highest severity first; ties broken by loopId so the view is stable across renders, never reshuffled. */ +const SEVERITY_RANK: Record = { high: 0, medium: 1, low: 2, none: 3 }; + +function compareRows(a: FleetLoopRow, b: FleetLoopRow): number { + const bySeverity = SEVERITY_RANK[a.escalation.severity] - SEVERITY_RANK[b.escalation.severity]; + return bySeverity !== 0 ? bySeverity : a.loopId.localeCompare(b.loopId); +} + +function toRow(loop: ActiveLoopFacts): FleetLoopRow { + const escalation = evaluateEscalation(loop); + return { + loopId: loop.loopId, + tenantId: loop.tenantId, + runStatus: loop.runStatus, + healthStatus: loop.healthStatus ?? "unknown", + needsAttention: escalation.shouldEscalate, + escalation, + }; +} + +/** + * Summarize the rented-loop fleet for the internal ops view (#4808). Pure: reads only the loops it is handed + * and returns a summary without mutating, fetching, or notifying anything. + * + * `needsAttention` is not a second opinion — each row's flag IS evaluateEscalation's own `shouldEscalate` for + * that loop, so the fleet view and the per-loop escalation path (#4806) can never disagree about what needs a + * human. `needingAttention` is those rows, worst-severity first, with ties broken by `loopId` so an operator + * watching the panel sees a stable order rather than rows reshuffling between renders. + * + * Both breakdowns are always fully populated (0 for an absent status/tier) so a panel binds to a fixed set of + * keys and never renders a hole. A loop with no health tier computed yet counts as `unknown` rather than being + * assumed healthy — an operator must be able to tell "nothing is wrong" from "nothing has checked yet". + */ +export function buildActiveLoopFleetSummary(loops: readonly ActiveLoopFacts[]): ActiveLoopFleetSummary { + const byStatus = Object.fromEntries(LOOP_RUN_STATUSES.map((s) => [s, 0])) as Record; + const byHealth = Object.fromEntries([...LOOP_HEALTH_TIERS, "unknown"].map((h) => [h, 0])) as Record; + + const rows = loops.map(toRow); + for (const row of rows) { + byStatus[row.runStatus] += 1; + byHealth[row.healthStatus] += 1; + } + + return { + activeCount: byStatus.running, + totalCount: rows.length, + byStatus, + byHealth, + needingAttention: rows.filter((row) => row.needsAttention).sort(compareRows), + loops: [...rows].sort(compareRows), + }; +} diff --git a/test/unit/loop-fleet-summary.test.ts b/test/unit/loop-fleet-summary.test.ts new file mode 100644 index 0000000000..bff7ca26c9 --- /dev/null +++ b/test/unit/loop-fleet-summary.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; + +import { evaluateEscalation } from "../../packages/loopover-engine/src/loop-escalation"; +import { + buildActiveLoopFleetSummary, + LOOP_HEALTH_TIERS, + LOOP_RUN_STATUSES, + type ActiveLoopFacts, +} from "../../packages/loopover-engine/src/loop-fleet-summary"; + +const loop = (over: Partial = {}): ActiveLoopFacts => ({ + loopId: "loop-1", + tenantId: "acme", + runStatus: "running", + healthStatus: "healthy", + ...over, +}); + +describe("buildActiveLoopFleetSummary (#4808)", () => { + // The acceptance criterion: an operator sees every currently active rented loop and its status at a glance. + it("counts the live fleet and breaks it down by run status and health", () => { + const summary = buildActiveLoopFleetSummary([ + loop({ loopId: "a" }), + loop({ loopId: "b" }), + loop({ loopId: "c", runStatus: "converged" }), // finished cleanly, so still healthy + loop({ loopId: "d", runStatus: "error", healthStatus: "critical" }), + loop({ loopId: "e", runStatus: "abandoned", healthStatus: "degraded" }), + ]); + + expect(summary.activeCount).toBe(2); // only the running ones are "currently active" + expect(summary.totalCount).toBe(5); + expect(summary.byStatus).toEqual({ running: 2, converged: 1, abandoned: 1, error: 1 }); + // Health is independent of run status: the converged loop is healthy too, so healthy counts a+b+c. + expect(summary.byHealth).toEqual({ critical: 1, degraded: 1, healthy: 3, unknown: 0 }); + }); + + it("an empty fleet totals to zeros, never undefined", () => { + const summary = buildActiveLoopFleetSummary([]); + expect(summary.activeCount).toBe(0); + expect(summary.totalCount).toBe(0); + expect(summary.needingAttention).toEqual([]); + expect(summary.loops).toEqual([]); + }); + + it("INVARIANT: every status and tier key is always present, so a panel never renders a hole", () => { + const summary = buildActiveLoopFleetSummary([]); + expect(Object.keys(summary.byStatus).sort()).toEqual([...LOOP_RUN_STATUSES].sort()); + expect(Object.keys(summary.byHealth).sort()).toEqual([...LOOP_HEALTH_TIERS, "unknown"].sort()); + expect(Object.values(summary.byStatus).every((n) => n === 0)).toBe(true); + expect(Object.values(summary.byHealth).every((n) => n === 0)).toBe(true); + }); + + it("a loop with no computed health tier reads as unknown, never assumed healthy", () => { + const summary = buildActiveLoopFleetSummary([loop({ healthStatus: undefined })]); + expect(summary.loops[0]!.healthStatus).toBe("unknown"); + expect(summary.byHealth.unknown).toBe(1); + expect(summary.byHealth.healthy).toBe(0); + }); + + describe("needsAttention", () => { + it("surfaces only the loops that need a human, worst-severity first", () => { + const summary = buildActiveLoopFleetSummary([ + loop({ loopId: "healthy-one" }), + loop({ loopId: "degraded-one", healthStatus: "degraded" }), // low → notify + loop({ loopId: "killed-one", killRequested: true }), // high → stop + loop({ loopId: "abandoned-one", runStatus: "abandoned" }), // medium → human_review + ]); + + expect(summary.needingAttention.map((r) => r.loopId)).toEqual(["killed-one", "abandoned-one", "degraded-one"]); + expect(summary.needingAttention.map((r) => r.escalation.severity)).toEqual(["high", "medium", "low"]); + expect(summary.needingAttention.every((r) => r.needsAttention)).toBe(true); + }); + + it("a fully healthy fleet needs no attention", () => { + const summary = buildActiveLoopFleetSummary([loop({ loopId: "a" }), loop({ loopId: "b" })]); + expect(summary.needingAttention).toEqual([]); + expect(summary.loops.every((r) => r.needsAttention === false)).toBe(true); + expect(summary.loops.every((r) => r.escalation.action === "none")).toBe(true); + }); + + // The point of reusing evaluateEscalation instead of re-deciding: the fleet view and the per-loop + // escalation path (#4806) cannot drift apart about what "needs a human" means. + it("INVARIANT: each row's decision IS evaluateEscalation's own, not a second opinion", () => { + const facts = loop({ loopId: "x", runStatus: "error", healthStatus: "critical", customerFlagged: true }); + const row = buildActiveLoopFleetSummary([facts]).loops[0]!; + expect(row.escalation).toEqual(evaluateEscalation(facts)); + expect(row.needsAttention).toBe(evaluateEscalation(facts).shouldEscalate); + }); + }); + + describe("stable ordering (an operator's panel must not reshuffle between renders)", () => { + it("breaks severity ties by loopId, deterministically", () => { + const summary = buildActiveLoopFleetSummary([ + loop({ loopId: "zeta", runStatus: "abandoned" }), + loop({ loopId: "alpha", runStatus: "abandoned" }), + loop({ loopId: "mid", runStatus: "abandoned" }), + ]); + expect(summary.needingAttention.map((r) => r.loopId)).toEqual(["alpha", "mid", "zeta"]); + }); + + it("input order never changes the output", () => { + const loops = [loop({ loopId: "b", healthStatus: "degraded" }), loop({ loopId: "a", runStatus: "error" }), loop({ loopId: "c" })]; + const forward = buildActiveLoopFleetSummary(loops); + const reversed = buildActiveLoopFleetSummary([...loops].reverse()); + expect(reversed).toEqual(forward); + }); + + it("does not mutate the caller's array", () => { + const loops = [loop({ loopId: "z" }), loop({ loopId: "a", runStatus: "error" })]; + const before = loops.map((l) => l.loopId); + buildActiveLoopFleetSummary(loops); + expect(loops.map((l) => l.loopId)).toEqual(before); + }); + }); + + it("keeps each loop attributed to its own tenant, so ops can see whose loop is misbehaving", () => { + const summary = buildActiveLoopFleetSummary([ + loop({ loopId: "a", tenantId: "acme", runStatus: "error" }), + loop({ loopId: "b", tenantId: "globex" }), + ]); + expect(summary.loops.find((r) => r.loopId === "a")!.tenantId).toBe("acme"); + expect(summary.loops.find((r) => r.loopId === "b")!.tenantId).toBe("globex"); + expect(summary.needingAttention.map((r) => r.tenantId)).toEqual(["acme"]); + }); +});