From b0e2c8161d099649077d584d46604afdc9df876d Mon Sep 17 00:00:00 2001 From: luciferlive112116 <291889058+luciferlive112116@users.noreply.github.com> Date: Wed, 15 Jul 2026 03:03:42 +0800 Subject: [PATCH] feat(mcp): evaluate when a rented loop should escalate to a human Closes #4806 Implements the support/escalation-path decision logic: given an already-computed loop outcome, health tier, and operator/customer signals, decide whether a loop needs a human and what action to take -- the deterministic core that routes "something's wrong" to a stop-and-review state. Composes with the loop-health evaluator (#4808) on the Rent-a-Loop path #4778. - new packages/loopover-engine/src/loop-escalation.ts (pure): evaluateEscalation(input) returns shouldEscalate + action (none/notify/human_review/stop) + severity + reasons, by precedence -- a requested stop wins; an errored run or critical health needs a human now; a give-up or customer flag needs a human soon; a soft degradation only notifies. No IO: it decides, the caller wires the action (a stop maps to #4809's kill-switch once that lands). - new loopover_evaluate_escalation MCP tool (src/mcp/server.ts); src/loop-escalation.ts is a thin re-export shim over the engine module. - tests cover every trigger and precedence tier plus the no-escalation path, at the engine level and end-to-end through the MCP tool. --- packages/loopover-engine/src/index.ts | 9 +++ .../loopover-engine/src/loop-escalation.ts | 62 +++++++++++++++++++ src/loop-escalation.ts | 5 ++ src/mcp/server.ts | 36 +++++++++++ test/unit/loop-escalation.test.ts | 46 ++++++++++++++ test/unit/mcp-loop-escalation.test.ts | 39 ++++++++++++ 6 files changed, 197 insertions(+) create mode 100644 packages/loopover-engine/src/loop-escalation.ts create mode 100644 src/loop-escalation.ts create mode 100644 test/unit/loop-escalation.test.ts create mode 100644 test/unit/mcp-loop-escalation.test.ts diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index ecfaa012c3..fd79585cc0 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -610,6 +610,15 @@ export { type LoopRunStatus, type ProgressSnapshot, } from "./loop-progress.js"; +export { + evaluateEscalation, + type EscalationAction, + type EscalationDecision, + type EscalationSeverity, + type LoopEscalationInput, + type LoopHealthTier, + type LoopRunOutcome, +} from "./loop-escalation.js"; export { buildMetadataRankInput, computeMetadataDupRisk, diff --git a/packages/loopover-engine/src/loop-escalation.ts b/packages/loopover-engine/src/loop-escalation.ts new file mode 100644 index 0000000000..5492b540c2 --- /dev/null +++ b/packages/loopover-engine/src/loop-escalation.ts @@ -0,0 +1,62 @@ +// Loop escalation evaluator (pure) — decides when a rented loop needs a human, and what action to take, so +// a support/escalation path can route "something's wrong" to a stop-and-review state (#4806, part of the +// Rent-a-Loop path #4778). Composes with the loop-health evaluator (#4808): it takes an already-computed +// run outcome + health tier + operator/customer signals and returns one deterministic escalation decision. +// No IO, no notifying, no stopping — it decides; the caller wires the action (a stop maps to #4809's +// kill-switch once that lands). Mirrors the quota (#4796) / loop-health (#4808) evaluator pattern. + +export type LoopRunOutcome = "running" | "converged" | "abandoned" | "error"; +export type LoopHealthTier = "healthy" | "degraded" | "critical"; +export type EscalationAction = "none" | "notify" | "human_review" | "stop"; +export type EscalationSeverity = "none" | "low" | "medium" | "high"; + +export type LoopEscalationInput = { + runStatus: LoopRunOutcome; + healthStatus?: LoopHealthTier | undefined; + /** The customer explicitly asked for help / review on their own loop. */ + customerFlagged?: boolean | undefined; + /** An operator (or the customer) requested a hard stop. */ + killRequested?: boolean | undefined; +}; + +export type EscalationDecision = { + shouldEscalate: boolean; + action: EscalationAction; + severity: EscalationSeverity; + reasons: string[]; +}; + +/** Decide whether — and how — a loop should be escalated to a human (#4806). Pure and deterministic. */ +export function evaluateEscalation(input: LoopEscalationInput): EscalationDecision { + // Independent reasons (never folded), so every triggering signal surfaces even when several fire at once. + const reasons: string[] = []; + if (input.killRequested === true) reasons.push("kill_requested"); + if (input.runStatus === "error") reasons.push("run_errored"); + if (input.healthStatus === "critical") reasons.push("health_critical"); + if (input.runStatus === "abandoned") reasons.push("run_abandoned"); + if (input.customerFlagged === true) reasons.push("customer_flagged"); + if (input.healthStatus === "degraded") reasons.push("health_degraded"); + + // Action + severity by precedence: a requested stop wins; a hard failure (errored/critical) needs a human + // now; a give-up/customer ask needs a human soon; a soft degradation only notifies. + let action: EscalationAction; + let severity: EscalationSeverity; + if (input.killRequested === true) { + action = "stop"; + severity = "high"; + } else if (input.runStatus === "error" || input.healthStatus === "critical") { + action = "human_review"; + severity = "high"; + } else if (input.runStatus === "abandoned" || input.customerFlagged === true) { + action = "human_review"; + severity = "medium"; + } else if (input.healthStatus === "degraded") { + action = "notify"; + severity = "low"; + } else { + action = "none"; + severity = "none"; + } + + return { shouldEscalate: action !== "none", action, severity, reasons }; +} diff --git a/src/loop-escalation.ts b/src/loop-escalation.ts new file mode 100644 index 0000000000..e579bb1a36 --- /dev/null +++ b/src/loop-escalation.ts @@ -0,0 +1,5 @@ +// Loop escalation evaluator (#4806) — thin re-export shim. The canonical implementation lives in +// `@loopover/engine` (packages/loopover-engine/src/loop-escalation.ts), imported via the relative source +// path (matching src/loop-progress.ts / src/loop-health.ts) so the published loopover-mcp / loopover-miner +// CLIs share one evaluator, and so this never depends on the engine's built dist/ during typecheck/test. +export * from "../packages/loopover-engine/src/loop-escalation"; diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 92faccbb07..162d91f479 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -158,6 +158,7 @@ import { buildSlopAssessment } from "../signals/slop"; import { validateIdeaSubmission, buildTaskGraph, buildClaimPlan } from "../idea-intake"; import { buildResultsPayload } from "../results-payload"; import { buildProgressSnapshot } from "../loop-progress"; +import { evaluateEscalation } from "../loop-escalation"; import { buildStructuralImprovementAssessment } from "../signals/improvement"; import { buildBoundaryTestGenerationFinding, buildBoundaryTestGenerationSpec } from "../signals/boundary-test-generation"; import { buildRepoDataQuality } from "../signals/data-quality"; @@ -999,6 +1000,21 @@ const buildProgressSnapshotOutputSchema = { done: z.boolean().optional(), }; +// Loop escalation evaluator input (#4806): an already-computed loop outcome + health tier + operator signals. +const evaluateEscalationShape = { + runStatus: z.enum(["running", "converged", "abandoned", "error"]), + healthStatus: z.enum(["healthy", "degraded", "critical"]).optional(), + customerFlagged: z.boolean().optional(), + killRequested: z.boolean().optional(), +}; + +const evaluateEscalationOutputSchema = { + shouldEscalate: z.boolean().optional(), + action: z.enum(["none", "notify", "human_review", "stop"]).optional(), + severity: z.enum(["none", "low", "medium", "high"]).optional(), + reasons: z.array(z.string()).optional(), +}; + // Deterministic structural-improvement counterpart to checkSlopRiskShape (#4746, sub-issue I of epic #4737): // the positive-axis mirror of checkSlopRisk, same pure local-metadata contract. changedFiles/tests/testFiles // are reused verbatim (same shape as checkSlopRiskShape) so the two signals never disagree about what counts @@ -1793,6 +1809,17 @@ export class LoopoverMcp { async (input) => this.toolResult(await this.buildLoopProgress(input)), ); + server.registerTool( + "loopover_evaluate_escalation", + { + description: + "Decide whether a rented loop needs a human, and what action to take (#4806), from an already-computed run outcome, health tier, and operator/customer signals — the deterministic support/escalation-path logic. Source-free; returns shouldEscalate + action (none/notify/human_review/stop) + severity + reasons. It decides; the caller wires the action.", + inputSchema: evaluateEscalationShape, + outputSchema: evaluateEscalationOutputSchema, + }, + async (input) => this.toolResult(await this.evalEscalation(input)), + ); + server.registerTool( "loopover_check_slop_risk", { @@ -3106,6 +3133,15 @@ export class LoopoverMcp { }; } + private async evalEscalation(input: z.infer>): Promise { + await this.enforceToolRateLimit("loopover_evaluate_escalation"); + const decision = evaluateEscalation(input); + return { + summary: `Escalation: ${decision.action} (severity ${decision.severity}), ${decision.reasons.length} reason(s).`, + data: decision as unknown as Record, + }; + } + private async buildLoopProgress(input: z.infer>): Promise { await this.enforceToolRateLimit("loopover_build_progress_snapshot"); const snapshot = buildProgressSnapshot(input); diff --git a/test/unit/loop-escalation.test.ts b/test/unit/loop-escalation.test.ts new file mode 100644 index 0000000000..3b3db4519b --- /dev/null +++ b/test/unit/loop-escalation.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { evaluateEscalation, type LoopEscalationInput } from "../../packages/loopover-engine/src/loop-escalation"; + +function decide(overrides: Partial = {}): ReturnType { + return evaluateEscalation({ runStatus: "running", ...overrides }); +} + +describe("evaluateEscalation (#4806)", () => { + it("does not escalate a healthy running loop", () => { + expect(decide({ healthStatus: "healthy" })).toEqual({ shouldEscalate: false, action: "none", severity: "none", reasons: [] }); + }); + + it("stops on an explicit kill request (highest precedence)", () => { + const d = decide({ killRequested: true }); + expect(d).toMatchObject({ shouldEscalate: true, action: "stop", severity: "high" }); + expect(d.reasons).toContain("kill_requested"); + }); + + it("routes an errored run to human review at high severity", () => { + expect(decide({ runStatus: "error" })).toMatchObject({ action: "human_review", severity: "high", reasons: ["run_errored"] }); + }); + + it("routes a critical health tier to human review at high severity", () => { + expect(decide({ healthStatus: "critical" })).toMatchObject({ action: "human_review", severity: "high", reasons: ["health_critical"] }); + }); + + it("routes an abandoned run to human review at medium severity", () => { + expect(decide({ runStatus: "abandoned" })).toMatchObject({ action: "human_review", severity: "medium", reasons: ["run_abandoned"] }); + }); + + it("routes a customer-flagged loop to human review at medium severity", () => { + expect(decide({ customerFlagged: true })).toMatchObject({ action: "human_review", severity: "medium", reasons: ["customer_flagged"] }); + }); + + it("only notifies on a soft degradation", () => { + expect(decide({ healthStatus: "degraded" })).toMatchObject({ action: "notify", severity: "low", reasons: ["health_degraded"] }); + }); + + it("surfaces every triggering reason and takes the highest-precedence action", () => { + const d = evaluateEscalation({ runStatus: "error", healthStatus: "degraded", customerFlagged: true, killRequested: true }); + expect(d.action).toBe("stop"); // kill request wins + expect(d.reasons).toEqual(expect.arrayContaining(["kill_requested", "run_errored", "customer_flagged", "health_degraded"])); + expect(d.reasons).not.toContain("run_abandoned"); // runStatus is error, not abandoned + expect(d.reasons).not.toContain("health_critical"); // health is degraded, not critical + }); +}); diff --git a/test/unit/mcp-loop-escalation.test.ts b/test/unit/mcp-loop-escalation.test.ts new file mode 100644 index 0000000000..c62d4cdf03 --- /dev/null +++ b/test/unit/mcp-loop-escalation.test.ts @@ -0,0 +1,39 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { describe, expect, it } from "vitest"; +import { LoopoverMcp } from "../../src/mcp/server"; +import { createTestEnv } from "../helpers/d1"; + +async function connect() { + const server = new LoopoverMcp(createTestEnv()).createServer(); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + const client = new Client({ name: "gittensory-escalation-test", version: "0.1.0" }, { capabilities: {} }); + await client.connect(clientTransport); + return client; +} + +describe("MCP loopover_evaluate_escalation", () => { + it("does not escalate a healthy loop", async () => { + const client = await connect(); + const result = await client.callTool({ + name: "loopover_evaluate_escalation", + arguments: { runStatus: "running", healthStatus: "healthy" }, + }); + expect(result.isError).toBeFalsy(); + const data = result.structuredContent as { shouldEscalate: boolean; action: string }; + expect(data.shouldEscalate).toBe(false); + expect(data.action).toBe("none"); + }); + + it("routes a failing loop to human review with reasons", async () => { + const client = await connect(); + const result = await client.callTool({ + name: "loopover_evaluate_escalation", + arguments: { runStatus: "error", healthStatus: "critical" }, + }); + const data = result.structuredContent as { shouldEscalate: boolean; action: string; severity: string; reasons: string[] }; + expect(data).toMatchObject({ shouldEscalate: true, action: "human_review", severity: "high" }); + expect(data.reasons).toEqual(expect.arrayContaining(["run_errored", "health_critical"])); + }); +});