Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/loopover-engine/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ export {
export {
runIterateLoop,
type IterateLoopDeps,
type IterateLoopShouldAbort,
type IterateLoopInput,
type IterateLoopIterationRecord,
type IterateLoopOutcome,
Expand Down
77 changes: 76 additions & 1 deletion packages/loopover-engine/src/miner/iterate-loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,15 @@ export type IterateLoopInput = {
rejectionSignaled: boolean;
};

/** Optional cooperative abort probed BEFORE every driver invocation (#5670). A bare `true` or
* `{ abort: true }` abandons with `kill_switch_engaged` without calling the driver for that iteration. */
export type IterateLoopShouldAbort =
| boolean
| {
abort: boolean;
reason?: string | undefined;
};

export type IterateLoopDeps = {
driver: CodingAgentDriver;
runSlopAssessment: SelfReviewAdapterDeps["runSlopAssessment"];
Expand All @@ -94,6 +103,8 @@ export type IterateLoopDeps = {
* injected-dependency discipline elsewhere (never a hardcoded `Date.now()` a test can't control). Defaults
* to the real `Date.now` when omitted. */
nowMs?: (() => number) | undefined;
/** Mid-iteration kill-switch / pause probe (#5670). Omitted = never abort mid-loop (pre-#5670 behavior). */
shouldAbort?: (() => IterateLoopShouldAbort) | undefined;
};

/** The terminal outcomes a full loop run can end in -- never `"continue"`, which is only ever a per-iteration,
Expand Down Expand Up @@ -186,10 +197,41 @@ function attemptLogEventTypeForDecision(decision: IterateLoopDecision): AttemptL
// reads as aborted; a genuine failure to converge (ceiling reached, or stuck with no progress) reads as
// failed. Both are still `action: "abandon"` in the decision itself -- this is only a coarser attempt-log
// classification layered on top, for the fixed six-value ATTEMPT_LOG_EVENT_TYPES vocabulary.
if (decision.abandonReason === "rejection_signaled" || decision.abandonReason === "self_review_ambiguous") return "attempt_aborted";
if (
decision.abandonReason === "rejection_signaled" ||
decision.abandonReason === "self_review_ambiguous" ||
decision.abandonReason === "kill_switch_engaged"
) {
return "attempt_aborted";
}
return "attempt_failed";
}

function resolveShouldAbort(deps: IterateLoopDeps): { abort: boolean; reason: string } {
if (typeof deps.shouldAbort !== "function") {
return { abort: false, reason: "" };
}
const raw = deps.shouldAbort();
if (typeof raw === "boolean") {
return {
abort: raw,
reason: raw
? "Kill-switch engaged mid-attempt; abandoning without starting another driver iteration."
: "",
};
}
if (raw && typeof raw === "object" && raw.abort === true) {
return {
abort: true,
reason:
typeof raw.reason === "string" && raw.reason.trim()
? raw.reason.trim()
: "Kill-switch engaged mid-attempt; abandoning without starting another driver iteration.",
};
}
return { abort: false, reason: "" };
}

/** A logging failure must never crash the loop or alter its decision -- mirrors the governor-ledger and
* pretooluse-hook append-failure handling elsewhere in this package. */
function safeAppendAttemptLogEvent(deps: IterateLoopDeps, event: AttemptLogEvent): void {
Expand Down Expand Up @@ -318,6 +360,39 @@ async function runIterateLoopCore(input: IterateLoopInput, deps: IterateLoopDeps
let totalCostUsd = 0;

for (let iterationNumber = 1; iterationNumber <= maxIterations; iterationNumber += 1) {
// Cooperative mid-iteration halt (#5670): probed BEFORE each driver call so a kill-switch that trips
// after iteration N prevents iteration N+1 (and prevents the first iteration when already tripped).
// Hard SIGKILL of an in-flight driver call is intentionally out of scope — matching #5437's budget
// abort, which also stops between iterations rather than interrupting a running LLM turn.
const abort = resolveShouldAbort(deps);
if (abort.abort) {
const decision: IterateLoopDecision = {
action: "abandon",
abandonReason: "kill_switch_engaged",
reason: abort.reason,
};
safeAppendAttemptLogEvent(deps, {
eventType: attemptLogEventTypeForDecision(decision),
attemptId: input.attemptId,
actionClass: "iterate_loop",
mode: input.mode,
reason: decision.reason,
payload: {
iterationNumber: iterationNumber - 1,
action: decision.action,
abandonReason: decision.abandonReason,
},
});
return {
outcome: "abandon",
finalDecision: decision,
iterationsUsed: iterationNumber - 1,
totalTurnsUsed,
totalCostUsd,
iterations,
};
}

const iterationStartMs = nowMs();
const driverResult = await runDriverSafely(input, deps, {
attemptId: input.attemptId,
Expand Down
10 changes: 9 additions & 1 deletion packages/loopover-engine/src/miner/iterate-policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,15 @@ export type IterateLoopAction = "continue" | "handoff" | "abandon";

/** Every distinct reason `decideNextAction` can abandon for -- kept as a closed literal union so a caller
* recording the decision (the attempt-log primitive, per #2333) has a stable, exhaustive vocabulary. */
export type AbandonReason = "rejection_signaled" | "self_review_ambiguous" | "max_iterations_reached" | "cost_ceiling_reached" | "no_progress";
export type AbandonReason =
| "rejection_signaled"
| "self_review_ambiguous"
| "max_iterations_reached"
| "cost_ceiling_reached"
| "no_progress"
/** Mid-attempt emergency stop (#5670): kill-switch (or operator pause acting as a stop signal) tripped
* between iterate-loop iterations — cooperative, not a hard SIGKILL of an in-flight driver call. */
| "kill_switch_engaged";

/**
* The self-review outcome as the policy needs it -- narrower than the full {@link SelfReviewVerdict} (self-
Expand Down
50 changes: 50 additions & 0 deletions packages/loopover-engine/test/iterate-loop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -406,3 +406,53 @@ test("a logging failure never crashes the loop or alters its decision", async ()

assert.equal(result.outcome, "handoff", "the tool call is still decided correctly even though every audit write failed");
});

test("abandon (kill_switch_engaged): shouldAbort before the first driver call abandons with zero iterations (#5670)", async () => {
let driverCalled = false;
const { deps, events } = collectingDeps({
driver: {
async run() {
driverCalled = true;
return okResult();
},
},
shouldAbort: () => true,
});
const result = await runIterateLoop(passingInput({ maxIterations: 3 }), deps);

assert.equal(result.outcome, "abandon");
assert.equal(result.finalDecision.abandonReason, "kill_switch_engaged");
assert.equal(result.iterationsUsed, 0);
assert.equal(driverCalled, false);
assert.equal(events.some((event) => event.eventType === "attempt_aborted"), true);
});

test("abandon (kill_switch_engaged): shouldAbort after iteration 1 prevents iteration 2 (#5670 mid-iteration)", async () => {
let probes = 0;
let callCount = 0;
// Same duplicate-PR fixture as the no_progress test: iter 1 fails predicted-gate and continues.
const pullRequests: PullRequestRecord[] = [openPr(42, "Retry uploads on 5xx responses", [7])];
const { deps } = collectingDeps({
driver: {
async run() {
callCount += 1;
return okResult(["src/upload.ts"], 2);
},
},
shouldAbort: () => {
probes += 1;
return probes > 1 ? { abort: true, reason: "operator tripped kill mid-run" } : false;
},
});
const result = await runIterateLoop(
passingInput({ maxIterations: 5, reviewContext: baseReviewContext({ pullRequests }) }),
deps,
);

assert.equal(result.outcome, "abandon");
assert.equal(result.finalDecision.abandonReason, "kill_switch_engaged");
assert.match(result.finalDecision.reason, /operator tripped kill mid-run/);
assert.equal(callCount, 1);
assert.equal(result.iterationsUsed, 1);
assert.equal(probes, 2);
});
45 changes: 42 additions & 3 deletions packages/loopover-miner/lib/attempt-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import { cleanupAttemptWorktree, prepareAttemptWorktree } from "./attempt-worktr
import { fetchSelfReviewContext } from "./self-review-context.js";
import { buildCodingTaskSpec } from "./coding-task-spec.js";
import { resolveAmsPolicy } from "./ams-policy.js";
import { checkMinerKillSwitch } from "./governor-kill-switch.js";
import { checkMinerKillSwitch, recordMinerKillSwitchTransition } from "./governor-kill-switch.js";
import { buildAttemptGovernorContext, buildAttemptLoopInput } from "./attempt-input-builder.js";
import { getAttemptHistory } from "./portfolio-queue.js";
import { loadReputationHistory, recordOwnSubmission } from "./governor-state.js";
Expand Down Expand Up @@ -388,7 +388,39 @@ export async function runAttempt(args, options = {}) {
const repoPaused = minerGoalSpec.spec.killSwitch.paused;

const checkKillSwitch = options.checkMinerKillSwitch ?? checkMinerKillSwitch;
const killSwitchScope = checkKillSwitch({ env, repoPaused }).scope;
const recordKillTransition = options.recordMinerKillSwitchTransition ?? recordMinerKillSwitchTransition;
let killSwitchScope = checkKillSwitch({ env, repoPaused }).scope;
let previousKillSwitchScope = killSwitchScope;

const resolveLiveKillSwitch = () => {
// Re-read the YAML flag each probe so an on-disk unpause/pause is reflected mid-attempt (#5670).
const liveRepoPaused = resolveGoalSpec(worktreeResult.repoPath).spec.killSwitch.paused;
const live = checkKillSwitch({ env, repoPaused: liveRepoPaused });
if (live.scope !== previousKillSwitchScope) {
try {
recordKillTransition({
repoFullName: parsed.repoFullName,
actionClass: "attempt",
previousScope: previousKillSwitchScope,
scope: live.scope,
});
} catch {
// Ledger append must never crash an aborting attempt.
}
previousKillSwitchScope = live.scope;
}
killSwitchScope = live.scope;
return live;
};

const shouldAbort = () => {
const live = resolveLiveKillSwitch();
if (!live.active) return false;
return {
abort: true,
reason: `Kill-switch (${live.scope}) engaged mid-attempt; abandoning without starting another driver iteration.`,
};
};

const loopInput = buildAttemptLoopInput({
codingTaskSpec,
Expand Down Expand Up @@ -435,7 +467,11 @@ export async function runAttempt(args, options = {}) {
submissionMode: amsPolicy.spec.submissionMode,
governor,
},
deps,
{
...deps,
shouldAbort,
resolveKillSwitchScope: () => resolveLiveKillSwitch().scope,
},
);

worktreeResult.attemptOk = result.outcome === "submitted";
Expand Down Expand Up @@ -510,6 +546,9 @@ export async function runAttempt(args, options = {}) {
// on any iteration this attempt ran, never fabricated.
totalTokensUsed: result.loopResult.finalMeterTotals.tokens,
iterationsUsed: result.loopResult.iterationsUsed,
...(result.outcome === "abandon" && result.loopResult.finalDecision?.abandonReason
? { abandonReason: result.loopResult.finalDecision.abandonReason }
: {}),
...("reason" in result ? { reason: result.reason } : {}),
...("decision" in result ? { decision: result.decision } : {}),
...("spec" in result ? { spec: result.spec } : {}),
Expand Down
4 changes: 4 additions & 0 deletions packages/loopover-miner/lib/attempt-runner.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ export type AttemptDeps = {
sessionStartMs?: number;
nowMs: number;
executeLocalWrite: (spec: LocalWriteActionSpec) => Promise<unknown>;
/** Mid-attempt kill-switch probe threaded into `runIterateLoop` (#5670). */
shouldAbort?: () => import("@loopover/engine").IterateLoopShouldAbort;
/** Live kill-switch scope resolver after handoff (#5670); defaults to the frozen attempt-start scope. */
resolveKillSwitchScope?: () => "global" | "repo" | "none";
};

export type AttemptResult =
Expand Down
25 changes: 25 additions & 0 deletions packages/loopover-miner/lib/attempt-runner.js
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ function assertInput(input) {
* sessionStartMs?: number,
* nowMs: number,
* executeLocalWrite: (spec: import("@loopover/engine").LocalWriteActionSpec) => Promise<unknown>,
* shouldAbort?: () => import("@loopover/engine").IterateLoopShouldAbort,
* resolveKillSwitchScope?: () => "global"|"repo"|"none",
* }} deps
*/
export async function runMinerAttempt(input, deps) {
Expand All @@ -109,6 +111,7 @@ export async function runMinerAttempt(input, deps) {
driver: deps.driver,
runSlopAssessment: deps.runSlopAssessment,
appendAttemptLogEvent: deps.appendAttemptLogEvent,
...(typeof deps.shouldAbort === "function" ? { shouldAbort: deps.shouldAbort } : {}),
});

if (loopResult.outcome === "abandon") {
Expand All @@ -117,6 +120,28 @@ export async function runMinerAttempt(input, deps) {

const handoffPacket = loopResult.handoffPacket;

// Re-check kill-switch AFTER handoff and BEFORE any write (#5670) when a live resolver is supplied.
// Without a live resolver, preserve pre-#5670 behavior: the frozen attempt-start scope is threaded into
// prepareOpenPrSubmission / the submission gate (which itself denies active kill scopes).
if (typeof deps.resolveKillSwitchScope === "function") {
const liveKillSwitchScope = deps.resolveKillSwitchScope();
if (liveKillSwitchScope !== "none") {
return {
outcome: "abandon",
loopResult: {
...loopResult,
outcome: "abandon",
finalDecision: {
action: "abandon",
abandonReason: "kill_switch_engaged",
reason: `Kill-switch (${liveKillSwitchScope}) engaged after handoff; refusing to open a PR.`,
},
handoffPacket: undefined,
},
};
}
}

const freshness = await checkSubmissionFreshness(
{ repoFullName: input.loopInput.repoFullName, issueNumber: input.issueNumber, minerLogin: input.minerLogin },
{ claimLedger: deps.claimLedger, fetchLiveIssueSnapshot: deps.fetchLiveIssueSnapshot, eventLedger: deps.eventLedger },
Expand Down
42 changes: 40 additions & 2 deletions packages/loopover-miner/lib/loop-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -311,14 +311,35 @@ export async function runLoop(args, options = {}) {
const killSwitch = checkKillSwitchFn({ env });
if (killSwitch.active) {
haltReason = `kill_switch_${killSwitch.scope}`;
cycles.push({ cycle: cycleIndex, outcome: "halted", reason: haltReason });
// Release the in-flight claim so left state is defined (#5670 / mirrors run-halt's markFailed).
if (claimed) {
portfolioQueue.markFailed(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl);
}
cycles.push({
cycle: cycleIndex,
outcome: "halted",
reason: haltReason,
...(claimed
? { repoFullName: claimed.repoFullName, identifier: claimed.identifier }
: {}),
});
break;
}

const pauseState = governorState.loadPauseState();
if (pauseState.paused) {
haltReason = "paused";
cycles.push({ cycle: cycleIndex, outcome: "halted", reason: haltReason });
if (claimed) {
portfolioQueue.markFailed(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl);
}
cycles.push({
cycle: cycleIndex,
outcome: "halted",
reason: haltReason,
...(claimed
? { repoFullName: claimed.repoFullName, identifier: claimed.identifier }
: {}),
});
break;
}

Expand Down Expand Up @@ -410,6 +431,9 @@ export async function runLoop(args, options = {}) {
// different iteration budget) and is requeued -- a genuinely stuck item is caught by non-convergence
// (reenqueues threshold) rather than silently retried forever.
const permanentBlock = attemptOutcome === "blocked_rejection_signaled";
// Mid-attempt kill-switch abandon (#5670): stop the outer loop immediately instead of waiting for the
// next between-cycle probe, and treat the item like any other re-queued abandon via markFailed below.
const killSwitchAbandon = lastResult?.abandonReason === "kill_switch_engaged";

if (submitted || permanentBlock) {
// Both terminal -- a submitted PR is done, and a repo-wide AI-usage-policy ban never resolves on retry --
Expand All @@ -421,6 +445,20 @@ export async function runLoop(args, options = {}) {
portfolioQueue.markFailed(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl);
}

if (killSwitchAbandon) {
const liveKill = checkKillSwitchFn({ env });
haltReason = liveKill.active ? `kill_switch_${liveKill.scope}` : "kill_switch_engaged";
cycles.push({
cycle: cycleIndex,
outcome: "halted",
reason: haltReason,
repoFullName: claimed.repoFullName,
identifier: claimed.identifier,
attemptOutcome,
});
break;
}

let reentryOutcome = "other";
let prNumber = null;
let prDisposition = null;
Expand Down
Loading