From bfadf32c92217ffd5899cdc8aec9a444b32b2261 Mon Sep 17 00:00:00 2001 From: leilei3167 Date: Thu, 24 Sep 2026 22:35:11 +0000 Subject: [PATCH 1/8] fix(automations): stop headless runs failing at the 5-minute tui-idle default Headless dispatch waited for tui-idle with no timeoutMs, so long agent runs inherited the runtime's 5-minute default and were marked dispatch_failed while still working. Observe completion through the retained-run terminal observer so timeouts re-arm until the shared 6h deadline. --- ...dless-dispatch-terminal-completion.test.ts | 104 ++++++++++++++++++ .../headless-dispatch-terminal-completion.ts | 20 ++++ src/main/startup/main-process-automations.ts | 33 ++---- 3 files changed, 132 insertions(+), 25 deletions(-) create mode 100644 src/main/automations/headless-dispatch-terminal-completion.test.ts create mode 100644 src/main/automations/headless-dispatch-terminal-completion.ts diff --git a/src/main/automations/headless-dispatch-terminal-completion.test.ts b/src/main/automations/headless-dispatch-terminal-completion.test.ts new file mode 100644 index 000000000000..6d21ae863eef --- /dev/null +++ b/src/main/automations/headless-dispatch-terminal-completion.test.ts @@ -0,0 +1,104 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createHeadlessAutomationCompletion } from './headless-dispatch-terminal-completion' +import { + createRuntimeAutomationRunTerminalObserver, + type AutomationRunTerminalHost +} from './runtime-terminal-run-observer' + +const HANDLE = 'terminal-1' +const RUNTIME_TUI_IDLE_TIMEOUT_MS = 5 * 60 * 1000 + +type FakePane = { + lastAgentStatus: 'idle' | 'working' | 'permission' | null +} + +type FakeWaiter = { + resolve: (value: { satisfied: boolean; blockedReason?: string }) => void + reject: (error: Error) => void + timer: ReturnType +} + +function createFakeRuntime(initial: Partial) { + const pane: FakePane = { + lastAgentStatus: null, + ...initial + } + const waiters = new Set() + + const satisfiedNow = (): boolean => pane.lastAgentStatus === 'idle' + + const runtime: AutomationRunTerminalHost & { + setPane: (next: Partial) => void + } = { + getTerminalHandleForPaneKey: () => HANDLE, + readTerminal: async () => ({ tail: ['agent finished after a long command'] }), + waitForTerminal: (_handle, options) => { + if (options?.signal?.aborted) { + return Promise.reject(new Error('request_aborted')) + } + if (satisfiedNow()) { + return Promise.resolve({ satisfied: true }) + } + return new Promise((resolve, reject) => { + const waiter: FakeWaiter = { + resolve, + reject, + timer: setTimeout(() => { + waiters.delete(waiter) + reject(new Error('timeout')) + }, options?.timeoutMs ?? RUNTIME_TUI_IDLE_TIMEOUT_MS) + } + waiters.add(waiter) + }) + }, + setPane: (next) => { + Object.assign(pane, next) + if (!satisfiedNow()) { + return + } + for (const waiter of waiters) { + waiters.delete(waiter) + clearTimeout(waiter.timer) + waiter.resolve({ satisfied: true }) + } + } + } + return runtime +} + +describe('createHeadlessAutomationCompletion (#22725)', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('documents the bare tui-idle wait that failed long runs at 5 minutes', async () => { + const runtime = createFakeRuntime({ lastAgentStatus: 'working' }) + const bare = runtime.waitForTerminal(HANDLE, { condition: 'tui-idle' }) + const expectation = expect(bare).rejects.toThrow('timeout') + await vi.advanceTimersByTimeAsync(RUNTIME_TUI_IDLE_TIMEOUT_MS) + await expectation + }) + + it('still completes after the agent works past the 5-minute tui-idle default', async () => { + const runtime = createFakeRuntime({ lastAgentStatus: 'working' }) + const observer = createRuntimeAutomationRunTerminalObserver(runtime) + const completion = createHeadlessAutomationCompletion({ + observeCompletion: (handle, options) => observer.observeCompletion(handle, options), + terminalHandle: HANDLE + }) + + // Agent keeps working past the runtime default that used to reject headless. + await vi.advanceTimersByTimeAsync(RUNTIME_TUI_IDLE_TIMEOUT_MS + 18_000) + runtime.setPane({ lastAgentStatus: 'idle' }) + await vi.advanceTimersByTimeAsync(10) + + await expect(completion).resolves.toMatchObject({ + status: 'completed', + error: null + }) + }) +}) diff --git a/src/main/automations/headless-dispatch-terminal-completion.ts b/src/main/automations/headless-dispatch-terminal-completion.ts new file mode 100644 index 000000000000..9c4cc6e66fee --- /dev/null +++ b/src/main/automations/headless-dispatch-terminal-completion.ts @@ -0,0 +1,20 @@ +import type { + AutomationRunCompletionObservation, + AutomationRunTerminalObserver +} from './run-completion-watcher' + +/** + * Headless dispatch used to call `waitForTerminal({ condition: 'tui-idle' })` + * with no `timeoutMs`, so it inherited the runtime's 5-minute default and marked + * longer agent runs as `dispatch_failed` / `timeout` while the agent was still + * working (#22725). Route through the same observer retained runs use so each + * tui-idle timeout re-arms until the shared 6h deadline. + */ +export function createHeadlessAutomationCompletion(args: { + observeCompletion: AutomationRunTerminalObserver['observeCompletion'] + terminalHandle: string +}): Promise { + return args.observeCompletion(args.terminalHandle, { + signal: new AbortController().signal + }) +} diff --git a/src/main/startup/main-process-automations.ts b/src/main/startup/main-process-automations.ts index a980c2720703..5fa05a4e1b85 100644 --- a/src/main/startup/main-process-automations.ts +++ b/src/main/startup/main-process-automations.ts @@ -1,5 +1,5 @@ import { AutomationService } from '../automations/service' -import { createHeadlessAutomationOutputSnapshotBuffer } from '../automations/headless-dispatch' +import { createHeadlessAutomationCompletion } from '../automations/headless-dispatch-terminal-completion' import { buildHeadlessAutomationWorktreeCreateArgs } from '../automations/headless-workspace-create' import { createRuntimeAutomationRunTerminalObserver } from '../automations/runtime-terminal-run-observer' import { mainProcessState as state } from './main-process-state' @@ -12,16 +12,16 @@ export function initializeMainProcessAutomations(): AutomationService { if (!store || !runtime || !claudeUsage || !codexUsage) { throw new Error('Runtime and usage stores must be initialized before automations') } + const terminalObserver = createRuntimeAutomationRunTerminalObserver(runtime) const service = new AutomationService(store, { claudeUsage, codexUsage, - terminalObserver: createRuntimeAutomationRunTerminalObserver(runtime), + terminalObserver, onAutomationsChanged: (payload) => runtime.notifyAutomationsChanged(payload), // Why: desktop clients mirror remote-host automations, but only a server process should execute remote_host_service-owned schedules. allowRemoteHostScheduling: state.isServeMode, headlessDispatcher: state.isServeMode ? async ({ automation, run, target }) => { - const terminalSnapshotLimit = 2_000 let terminalHandle: string let terminalSessionId: string | null = null let terminalPaneKey: string | null = null @@ -61,28 +61,11 @@ export function initializeMainProcessAutomations(): AutomationService { const worktree = await runtime.showManagedWorktree(`id:${workspaceId}`) workspaceDisplayName = worktree.displayName ?? null } - const completion = (async () => { - const wait = await runtime.waitForTerminal(terminalHandle, { condition: 'tui-idle' }) - const read = await runtime.readTerminal(terminalHandle, { - limit: terminalSnapshotLimit - }) - const snapshotBuffer = createHeadlessAutomationOutputSnapshotBuffer() - snapshotBuffer.append(read.tail.join('\n')) - if (wait.satisfied) { - return { - status: 'completed' as const, - outputSnapshot: snapshotBuffer.snapshot(), - error: null - } - } - return { - status: 'dispatch_failed' as const, - outputSnapshot: snapshotBuffer.snapshot(), - error: wait.blockedReason - ? `Automation agent is blocked: ${wait.blockedReason}.` - : 'Automation agent did not report completion.' - } - })() + const completion = createHeadlessAutomationCompletion({ + observeCompletion: (handle, options) => + terminalObserver.observeCompletion(handle, options), + terminalHandle + }) return { workspaceId, workspaceDisplayName, From 2f6359c0aaf882e6b6102d25f47cf81ce39212ab Mon Sep 17 00:00:00 2001 From: leilei3167 Date: Fri, 25 Sep 2026 00:10:20 +0000 Subject: [PATCH 2/8] fix(automations): observe headless completion before worktree lookup Start the shared terminal observer as soon as the reused-workspace agent terminal exists, before awaiting showManagedWorktree, so a fast finish is not mistaken for leftover idle. Route the unit test through the production helper instead of a bare wait mock. --- ...dless-dispatch-terminal-completion.test.ts | 22 ++++++++++++++----- src/main/startup/main-process-automations.ts | 15 +++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/main/automations/headless-dispatch-terminal-completion.test.ts b/src/main/automations/headless-dispatch-terminal-completion.test.ts index 6d21ae863eef..1601c4732f2b 100644 --- a/src/main/automations/headless-dispatch-terminal-completion.test.ts +++ b/src/main/automations/headless-dispatch-terminal-completion.test.ts @@ -75,12 +75,22 @@ describe('createHeadlessAutomationCompletion (#22725)', () => { vi.useRealTimers() }) - it('documents the bare tui-idle wait that failed long runs at 5 minutes', async () => { - const runtime = createFakeRuntime({ lastAgentStatus: 'working' }) - const bare = runtime.waitForTerminal(HANDLE, { condition: 'tui-idle' }) - const expectation = expect(bare).rejects.toThrow('timeout') - await vi.advanceTimersByTimeAsync(RUNTIME_TUI_IDLE_TIMEOUT_MS) - await expectation + it('routes headless completion through observeCompletion', async () => { + const observeCompletion = vi.fn(async () => ({ + status: 'completed' as const, + outputSnapshot: null, + error: null + })) + await expect( + createHeadlessAutomationCompletion({ + observeCompletion, + terminalHandle: HANDLE + }) + ).resolves.toMatchObject({ status: 'completed', error: null }) + expect(observeCompletion).toHaveBeenCalledWith( + HANDLE, + expect.objectContaining({ signal: expect.any(AbortSignal) }) + ) }) it('still completes after the agent works past the 5-minute tui-idle default', async () => { diff --git a/src/main/startup/main-process-automations.ts b/src/main/startup/main-process-automations.ts index 5fa05a4e1b85..9e6b4fe77524 100644 --- a/src/main/startup/main-process-automations.ts +++ b/src/main/startup/main-process-automations.ts @@ -58,8 +58,23 @@ export function initializeMainProcessAutomations(): AutomationService { terminalPaneKey = terminal.paneKey ?? null terminalPtyId = terminal.ptyId ?? null workspaceId = terminal.worktreeId + // Attach before showManagedWorktree: a reused pane can finish during that + // await, and the shared observer would treat leftover idle as "never started". + const completion = createHeadlessAutomationCompletion({ + observeCompletion: (handle, options) => + terminalObserver.observeCompletion(handle, options), + terminalHandle + }) const worktree = await runtime.showManagedWorktree(`id:${workspaceId}`) workspaceDisplayName = worktree.displayName ?? null + return { + workspaceId, + workspaceDisplayName, + terminalSessionId, + terminalPaneKey, + terminalPtyId, + completion + } } const completion = createHeadlessAutomationCompletion({ observeCompletion: (handle, options) => From 30f03c1c37526a93950431f050dd4b4dc93c0330 Mon Sep 17 00:00:00 2001 From: leilei3167 Date: Fri, 25 Sep 2026 04:20:13 +0000 Subject: [PATCH 3/8] fix(automations): settle headless completion before worktree await Attach a no-op rejection handler when creating the headless completion promise so an early observe failure during showManagedWorktree does not surface as main_unhandled_rejection before the dispatcher returns. Also return both workspace modes through one launch-result object. --- src/main/startup/main-process-automations.ts | 26 ++++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/main/startup/main-process-automations.ts b/src/main/startup/main-process-automations.ts index 9e6b4fe77524..5e2d377c337f 100644 --- a/src/main/startup/main-process-automations.ts +++ b/src/main/startup/main-process-automations.ts @@ -1,6 +1,7 @@ import { AutomationService } from '../automations/service' import { createHeadlessAutomationCompletion } from '../automations/headless-dispatch-terminal-completion' import { buildHeadlessAutomationWorktreeCreateArgs } from '../automations/headless-workspace-create' +import type { AutomationRunCompletionObservation } from '../automations/run-completion-watcher' import { createRuntimeAutomationRunTerminalObserver } from '../automations/runtime-terminal-run-observer' import { mainProcessState as state } from './main-process-state' @@ -58,29 +59,22 @@ export function initializeMainProcessAutomations(): AutomationService { terminalPaneKey = terminal.paneKey ?? null terminalPtyId = terminal.ptyId ?? null workspaceId = terminal.worktreeId - // Attach before showManagedWorktree: a reused pane can finish during that - // await, and the shared observer would treat leftover idle as "never started". - const completion = createHeadlessAutomationCompletion({ + } + // Attach before showManagedWorktree on the reused path: a pane can finish + // during that await, and the shared observer would treat leftover idle as + // "never started". Swallow rejections until the dispatcher returns so the + // runner can attach its own handler without a main_unhandled_rejection. + const completion: Promise = + createHeadlessAutomationCompletion({ observeCompletion: (handle, options) => terminalObserver.observeCompletion(handle, options), terminalHandle }) + void completion.catch(() => {}) + if (automation.workspaceMode !== 'new_per_run') { const worktree = await runtime.showManagedWorktree(`id:${workspaceId}`) workspaceDisplayName = worktree.displayName ?? null - return { - workspaceId, - workspaceDisplayName, - terminalSessionId, - terminalPaneKey, - terminalPtyId, - completion - } } - const completion = createHeadlessAutomationCompletion({ - observeCompletion: (handle, options) => - terminalObserver.observeCompletion(handle, options), - terminalHandle - }) return { workspaceId, workspaceDisplayName, From 970e45c6aca5c591f3a24cba80ffbe12935da1d0 Mon Sep 17 00:00:00 2001 From: leilei3167 Date: Fri, 25 Sep 2026 08:27:35 +0000 Subject: [PATCH 4/8] fix(automations): treat exited terminals as completion, not stale idle Skip the busy-edge guard when the initial tui-idle probe already reports status exited, and only classify exit code 0 as completed. --- ...dless-dispatch-terminal-completion.test.ts | 27 ++++++++- .../runtime-terminal-run-observer.test.ts | 57 +++++++++++++++++-- .../runtime-terminal-run-observer.ts | 11 ++-- 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/src/main/automations/headless-dispatch-terminal-completion.test.ts b/src/main/automations/headless-dispatch-terminal-completion.test.ts index 1601c4732f2b..cbfc787c9d06 100644 --- a/src/main/automations/headless-dispatch-terminal-completion.test.ts +++ b/src/main/automations/headless-dispatch-terminal-completion.test.ts @@ -12,8 +12,17 @@ type FakePane = { lastAgentStatus: 'idle' | 'working' | 'permission' | null } +type FakeWaitResult = { + handle: string + condition: 'tui-idle' + satisfied: boolean + status: 'running' | 'exited' | 'unknown' + exitCode: number | null + blockedReason?: string +} + type FakeWaiter = { - resolve: (value: { satisfied: boolean; blockedReason?: string }) => void + resolve: (value: FakeWaitResult) => void reject: (error: Error) => void timer: ReturnType } @@ -37,7 +46,13 @@ function createFakeRuntime(initial: Partial) { return Promise.reject(new Error('request_aborted')) } if (satisfiedNow()) { - return Promise.resolve({ satisfied: true }) + return Promise.resolve({ + handle: HANDLE, + condition: 'tui-idle', + satisfied: true, + status: 'running', + exitCode: null + }) } return new Promise((resolve, reject) => { const waiter: FakeWaiter = { @@ -59,7 +74,13 @@ function createFakeRuntime(initial: Partial) { for (const waiter of waiters) { waiters.delete(waiter) clearTimeout(waiter.timer) - waiter.resolve({ satisfied: true }) + waiter.resolve({ + handle: HANDLE, + condition: 'tui-idle', + satisfied: true, + status: 'running', + exitCode: null + }) } } } diff --git a/src/main/automations/runtime-terminal-run-observer.test.ts b/src/main/automations/runtime-terminal-run-observer.test.ts index a9b307c32984..bc2e6e39dea7 100644 --- a/src/main/automations/runtime-terminal-run-observer.test.ts +++ b/src/main/automations/runtime-terminal-run-observer.test.ts @@ -13,10 +13,21 @@ type FakePane = { lastAgentStatus: 'idle' | 'working' | 'permission' | null paneTitle: string | null preview: string + status: 'running' | 'exited' | 'unknown' + exitCode: number | null +} + +type FakeWaitResult = { + handle: string + condition: 'tui-idle' + satisfied: boolean + status: FakePane['status'] + exitCode: number | null + blockedReason?: string } type FakeWaiter = { - resolve: (value: { satisfied: boolean; blockedReason?: string }) => void + resolve: (value: FakeWaitResult) => void reject: (error: Error) => void timer: ReturnType } @@ -26,6 +37,8 @@ function createFakeRuntime(initial: Partial) { lastAgentStatus: null, paneTitle: null, preview: '', + status: 'running', + exitCode: null, ...initial } const waiters = new Set() @@ -36,7 +49,16 @@ function createFakeRuntime(initial: Partial) { const satisfiedNow = (): boolean => pane.lastAgentStatus === 'idle' || (pane.paneTitle?.toLowerCase().includes('idle') ?? false) || - pane.preview.trimEnd().endsWith('$') + pane.preview.trimEnd().endsWith('$') || + pane.status === 'exited' + + const waitResult = (satisfied: boolean): FakeWaitResult => ({ + handle: HANDLE, + condition: 'tui-idle', + satisfied, + status: pane.status, + exitCode: pane.exitCode + }) const runtime: AutomationRunTerminalHost & { setPane: (next: Partial) => void @@ -50,7 +72,7 @@ function createFakeRuntime(initial: Partial) { return Promise.reject(new Error('request_aborted')) } if (satisfiedNow()) { - return Promise.resolve({ satisfied: true }) + return Promise.resolve(waitResult(true)) } return new Promise((resolve, reject) => { const waiter: FakeWaiter = { @@ -72,7 +94,7 @@ function createFakeRuntime(initial: Partial) { for (const waiter of waiters) { waiters.delete(waiter) clearTimeout(waiter.timer) - waiter.resolve({ satisfied: true }) + waiter.resolve(waitResult(true)) } }, waitCalls: () => waitCalls @@ -192,4 +214,31 @@ describe('createRuntimeAutomationRunTerminalObserver', () => { expect(runtime.waitCalls()).toBeLessThanOrEqual(80) await run.promise }) + + it('completes a fast-finished new-per-run terminal that is already exited', async () => { + const runtime = createFakeRuntime({ + lastAgentStatus: 'idle', + status: 'exited', + exitCode: 0 + }) + const run = observe(runtime) + + await vi.advanceTimersByTimeAsync(1_000) + expect(run.settled[0]?.status).toBe('completed') + await run.promise + }) + + it('fails an exited terminal with a non-zero exit code', async () => { + const runtime = createFakeRuntime({ + lastAgentStatus: 'idle', + status: 'exited', + exitCode: 1 + }) + const run = observe(runtime) + + await vi.advanceTimersByTimeAsync(1_000) + expect(run.settled[0]?.status).toBe('dispatch_failed') + await run.promise + }) + }) diff --git a/src/main/automations/runtime-terminal-run-observer.ts b/src/main/automations/runtime-terminal-run-observer.ts index 0dc6750e8de1..3998d63fad08 100644 --- a/src/main/automations/runtime-terminal-run-observer.ts +++ b/src/main/automations/runtime-terminal-run-observer.ts @@ -4,6 +4,7 @@ import type { AutomationRunTerminalObserver } from './run-completion-watcher' import type { AutomationRunOutputSnapshot } from '../../shared/automations-types' +import type { RuntimeTerminalWait } from '../../shared/runtime-types' const TERMINAL_SNAPSHOT_LIMIT = 2_000 @@ -30,7 +31,7 @@ export type AutomationRunTerminalHost = { waitForTerminal( handle: string, options?: { condition?: 'tui-idle'; timeoutMs?: number; signal?: AbortSignal } - ): Promise<{ satisfied: boolean; blockedReason?: string }> + ): Promise readTerminal(handle: string, opts?: { limit?: number }): Promise<{ tail: string[] }> } @@ -73,7 +74,9 @@ async function isTuiIdleSatisfiedNow( signal }) // A blocked pane is not "already finished"; let the real wait report it. - return wait.satisfied + // An exited terminal can satisfy tui-idle while status is 'exited'; treat that + // as completion evidence, not a stale idle that needs a busy edge. + return wait.satisfied && wait.status !== 'exited' } catch (error) { if (isTerminalWaitTimeout(error)) { return false @@ -117,10 +120,10 @@ async function readTerminalSnapshot( async function buildObservation( runtime: AutomationRunTerminalHost, handle: string, - wait: { satisfied: boolean; blockedReason?: string } + wait: RuntimeTerminalWait ): Promise { const outputSnapshot = await readTerminalSnapshot(runtime, handle) - if (wait.satisfied) { + if (wait.satisfied && (wait.status !== 'exited' || wait.exitCode === 0)) { return { status: 'completed', outputSnapshot, error: null } } return { From d5ecb5051758c4422e32f88d4c7b406f466c72d1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 25 Sep 2026 12:15:00 +0000 Subject: [PATCH 5/8] test(automations): align terminal wait fakes with RuntimeTerminalWait --- .../automations/headless-dispatch-terminal-completion.test.ts | 1 - src/main/automations/runtime-terminal-run-observer.test.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/main/automations/headless-dispatch-terminal-completion.test.ts b/src/main/automations/headless-dispatch-terminal-completion.test.ts index cbfc787c9d06..dc9b8d2aaca6 100644 --- a/src/main/automations/headless-dispatch-terminal-completion.test.ts +++ b/src/main/automations/headless-dispatch-terminal-completion.test.ts @@ -18,7 +18,6 @@ type FakeWaitResult = { satisfied: boolean status: 'running' | 'exited' | 'unknown' exitCode: number | null - blockedReason?: string } type FakeWaiter = { diff --git a/src/main/automations/runtime-terminal-run-observer.test.ts b/src/main/automations/runtime-terminal-run-observer.test.ts index bc2e6e39dea7..8c7777da4491 100644 --- a/src/main/automations/runtime-terminal-run-observer.test.ts +++ b/src/main/automations/runtime-terminal-run-observer.test.ts @@ -23,7 +23,6 @@ type FakeWaitResult = { satisfied: boolean status: FakePane['status'] exitCode: number | null - blockedReason?: string } type FakeWaiter = { @@ -240,5 +239,4 @@ describe('createRuntimeAutomationRunTerminalObserver', () => { expect(run.settled[0]?.status).toBe('dispatch_failed') await run.promise }) - }) From 31a88e005a225e8ec8be1b959ee067d343b71265 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 25 Sep 2026 16:21:55 +0000 Subject: [PATCH 6/8] fix(automations): abort headless completion when service stops Wire headless terminal observation through AutomationService-owned AbortControllers so stop() cancels the 6h observer path and skips persisting dispatch results after shutdown. --- .../automations/headless-completion-abort.ts | 50 +++++++++++++++++++ .../automations/headless-dispatch-runner.ts | 29 ++++++++--- ...dless-dispatch-terminal-completion.test.ts | 6 ++- .../headless-dispatch-terminal-completion.ts | 3 +- src/main/automations/headless-dispatch.ts | 3 ++ .../run-completion-watcher.test.ts | 27 ++++++++++ src/main/automations/service.ts | 7 ++- src/main/startup/main-process-automations.ts | 5 +- 8 files changed, 118 insertions(+), 12 deletions(-) create mode 100644 src/main/automations/headless-completion-abort.ts diff --git a/src/main/automations/headless-completion-abort.ts b/src/main/automations/headless-completion-abort.ts new file mode 100644 index 000000000000..9299687f8516 --- /dev/null +++ b/src/main/automations/headless-completion-abort.ts @@ -0,0 +1,50 @@ +import type { + HeadlessAutomationDispatcher, + HeadlessAutomationDispatchLaunch +} from './headless-dispatch' + +/** Tracks headless completion observers so AutomationService.stop() can abort them. */ +export class HeadlessCompletionAbortRegistry { + private readonly controllers = new Map() + + wrapDispatcher(inner: HeadlessAutomationDispatcher): HeadlessAutomationDispatcher { + return async (request) => { + const controller = new AbortController() + this.controllers.set(request.run.id, controller) + try { + const launch = await inner({ + ...request, + completionSignal: controller.signal + }) + this.trackCompletion(request.run.id, controller, launch) + return { ...launch, completionAbortSignal: controller.signal } + } catch (error) { + this.controllers.delete(request.run.id) + throw error + } + } + } + + abortAll(): void { + for (const controller of this.controllers.values()) { + controller.abort() + } + this.controllers.clear() + } + + private trackCompletion( + runId: string, + controller: AbortController, + launch: HeadlessAutomationDispatchLaunch + ): void { + if (!launch.completion) { + this.controllers.delete(runId) + return + } + void launch.completion.finally(() => { + if (this.controllers.get(runId) === controller) { + this.controllers.delete(runId) + } + }) + } +} diff --git a/src/main/automations/headless-dispatch-runner.ts b/src/main/automations/headless-dispatch-runner.ts index 8d4e3aed290d..ba0911fe9190 100644 --- a/src/main/automations/headless-dispatch-runner.ts +++ b/src/main/automations/headless-dispatch-runner.ts @@ -27,6 +27,16 @@ function describeDispatchError(error: unknown): string { return error instanceof Error ? error.message : String(error) } +function shouldIgnoreHeadlessCompletion( + error: unknown, + abortSignal: AbortSignal | undefined +): boolean { + if (abortSignal?.aborted) { + return true + } + return error instanceof Error && error.message === 'request_aborted' +} + export async function runHeadlessAutomationDispatch( ctx: HeadlessAutomationDispatchContext ): Promise { @@ -63,9 +73,13 @@ export async function runHeadlessAutomationDispatch( ctx.watchRun(updated) return updated } + const completionAbortSignal = launch.completionAbortSignal void launch.completion - .then((completion) => - ctx.markDispatchResult({ + .then((completion) => { + if (completionAbortSignal?.aborted) { + return + } + return ctx.markDispatchResult({ runId: run.id, status: completion.status, ...launchRunTarget, @@ -73,15 +87,18 @@ export async function runHeadlessAutomationDispatch( outputSnapshot: completion.outputSnapshot ?? null, error: completion.error ?? null }) - ) - .catch((error) => - ctx.markDispatchResult({ + }) + .catch((error) => { + if (shouldIgnoreHeadlessCompletion(error, completionAbortSignal)) { + return + } + return ctx.markDispatchResult({ runId: run.id, status: 'dispatch_failed', ...launchRunTarget, error: describeDispatchError(error) }) - ) + }) return updated } catch (error) { return runs.updateRun({ diff --git a/src/main/automations/headless-dispatch-terminal-completion.test.ts b/src/main/automations/headless-dispatch-terminal-completion.test.ts index dc9b8d2aaca6..129b8c578051 100644 --- a/src/main/automations/headless-dispatch-terminal-completion.test.ts +++ b/src/main/automations/headless-dispatch-terminal-completion.test.ts @@ -104,7 +104,8 @@ describe('createHeadlessAutomationCompletion (#22725)', () => { await expect( createHeadlessAutomationCompletion({ observeCompletion, - terminalHandle: HANDLE + terminalHandle: HANDLE, + signal: new AbortController().signal }) ).resolves.toMatchObject({ status: 'completed', error: null }) expect(observeCompletion).toHaveBeenCalledWith( @@ -118,7 +119,8 @@ describe('createHeadlessAutomationCompletion (#22725)', () => { const observer = createRuntimeAutomationRunTerminalObserver(runtime) const completion = createHeadlessAutomationCompletion({ observeCompletion: (handle, options) => observer.observeCompletion(handle, options), - terminalHandle: HANDLE + terminalHandle: HANDLE, + signal: new AbortController().signal }) // Agent keeps working past the runtime default that used to reject headless. diff --git a/src/main/automations/headless-dispatch-terminal-completion.ts b/src/main/automations/headless-dispatch-terminal-completion.ts index 9c4cc6e66fee..f94c74d01f0e 100644 --- a/src/main/automations/headless-dispatch-terminal-completion.ts +++ b/src/main/automations/headless-dispatch-terminal-completion.ts @@ -13,8 +13,9 @@ import type { export function createHeadlessAutomationCompletion(args: { observeCompletion: AutomationRunTerminalObserver['observeCompletion'] terminalHandle: string + signal: AbortSignal }): Promise { return args.observeCompletion(args.terminalHandle, { - signal: new AbortController().signal + signal: args.signal }) } diff --git a/src/main/automations/headless-dispatch.ts b/src/main/automations/headless-dispatch.ts index 359673bc573e..6d23ca3cade9 100644 --- a/src/main/automations/headless-dispatch.ts +++ b/src/main/automations/headless-dispatch.ts @@ -18,12 +18,15 @@ export type HeadlessAutomationDispatchLaunch = { outputSnapshot?: AutomationRunOutputSnapshot | null error?: string | null }> + /** Set by AutomationService so headless completion can be dropped after stop(). */ + completionAbortSignal?: AbortSignal } export type HeadlessAutomationDispatcher = (request: { automation: Automation run: AutomationRun target: Extract + completionSignal: AbortSignal }) => Promise export function createHeadlessAutomationOutputSnapshotBuffer(): { diff --git a/src/main/automations/run-completion-watcher.test.ts b/src/main/automations/run-completion-watcher.test.ts index 31c1aea0bb47..3e694982f5ea 100644 --- a/src/main/automations/run-completion-watcher.test.ts +++ b/src/main/automations/run-completion-watcher.test.ts @@ -273,6 +273,33 @@ describe('authority-owned automation run completion', () => { service.stop() }) + it('aborts in-flight headless completion on stop without persisting a terminal status', async () => { + const store = await createStore() + const automation = createAutomation(store) + let completionSignal: AbortSignal | null = null + let resolveCompletion!: (value: AutomationRunCompletionObservation) => void + const service = new AutomationService(store, { + headlessDispatcher: async ({ completionSignal: signal }) => { + completionSignal = signal + return { + ...LAUNCH_TARGET, + completion: new Promise((resolve) => { + resolveCompletion = resolve + }) + } + } + }) + + const run = await service.runNow(automation.id) + expect(readRun(store, automation.id, run.id).status).toBe('dispatched') + service.stop() + expect(completionSignal?.aborted).toBe(true) + resolveCompletion({ status: 'completed', error: null }) + await Promise.resolve() + await Promise.resolve() + expect(readRun(store, automation.id, run.id).status).toBe('dispatched') + }) + it('disposes watchers on stop', async () => { const store = await createStore() const automation = createAutomation(store) diff --git a/src/main/automations/service.ts b/src/main/automations/service.ts index 227a290783bf..ee745bdc6744 100644 --- a/src/main/automations/service.ts +++ b/src/main/automations/service.ts @@ -18,6 +18,7 @@ import { runAutomationPrecheck } from './precheck-runner' import { resolveAutomationRunTarget, type AutomationRunTargetResult } from './run-target-resolution' import { writeAutomationRunUsage } from './run-usage-collection' import type { HeadlessAutomationDispatcher } from './headless-dispatch' +import { HeadlessCompletionAbortRegistry } from './headless-completion-abort' import { clearAutomationDispatchTokens, createAutomationDispatchToken } from './dispatch-tokens' import { runHeadlessAutomationDispatch } from './headless-dispatch-runner' import { @@ -53,6 +54,7 @@ export class AutomationService { private readonly codexUsage: CodexUsageStore | null private readonly allowRemoteHostScheduling: boolean private readonly headlessDispatcher: HeadlessAutomationDispatcher | null + private readonly headlessCompletionAbort = new HeadlessCompletionAbortRegistry() private readonly publish: PublishAutomationsChanged | null private readonly runs: AutomationRunWriter private readonly completionWatcher: AutomationRunCompletionWatcher | null @@ -78,7 +80,9 @@ export class AutomationService { this.claudeUsage = opts.claudeUsage ?? null this.codexUsage = opts.codexUsage ?? null this.allowRemoteHostScheduling = opts.allowRemoteHostScheduling ?? false - this.headlessDispatcher = opts.headlessDispatcher ?? null + this.headlessDispatcher = opts.headlessDispatcher + ? this.headlessCompletionAbort.wrapDispatcher(opts.headlessDispatcher) + : null this.publish = opts.onAutomationsChanged ?? null this.runs = createAutomationRunWriter(store, this.publish) this.completionWatcher = opts.terminalObserver @@ -131,6 +135,7 @@ export class AutomationService { stop(): void { this.completionWatcher?.dispose() + this.headlessCompletionAbort.abortAll() if (!this.timer) { return } diff --git a/src/main/startup/main-process-automations.ts b/src/main/startup/main-process-automations.ts index 5e2d377c337f..5bca2978e17b 100644 --- a/src/main/startup/main-process-automations.ts +++ b/src/main/startup/main-process-automations.ts @@ -22,7 +22,7 @@ export function initializeMainProcessAutomations(): AutomationService { // Why: desktop clients mirror remote-host automations, but only a server process should execute remote_host_service-owned schedules. allowRemoteHostScheduling: state.isServeMode, headlessDispatcher: state.isServeMode - ? async ({ automation, run, target }) => { + ? async ({ automation, run, target, completionSignal }) => { let terminalHandle: string let terminalSessionId: string | null = null let terminalPaneKey: string | null = null @@ -68,7 +68,8 @@ export function initializeMainProcessAutomations(): AutomationService { createHeadlessAutomationCompletion({ observeCompletion: (handle, options) => terminalObserver.observeCompletion(handle, options), - terminalHandle + terminalHandle, + signal: completionSignal }) void completion.catch(() => {}) if (automation.workspaceMode !== 'new_per_run') { From e387f24f46950a8f927f182ddc21f870aabad7ce Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 25 Sep 2026 20:22:41 +0000 Subject: [PATCH 7/8] fix(automations): wire completionSignal through headless dispatch runner Begin completion abort before dispatch, pass the signal into the wrapped dispatcher, and swallow completion cleanup rejections after stop(). --- .../automations/headless-completion-abort.ts | 29 ++++++++++++------- .../automations/headless-dispatch-runner.ts | 9 +++++- .../retained-run-reconciliation.test.ts | 5 +++- .../run-completion-watcher.test.ts | 22 ++++++++------ src/main/automations/service-precheck.test.ts | 1 + src/main/automations/service.ts | 1 + 6 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/main/automations/headless-completion-abort.ts b/src/main/automations/headless-completion-abort.ts index 9299687f8516..c1df01564785 100644 --- a/src/main/automations/headless-completion-abort.ts +++ b/src/main/automations/headless-completion-abort.ts @@ -7,15 +7,20 @@ import type { export class HeadlessCompletionAbortRegistry { private readonly controllers = new Map() + beginRun(runId: string): AbortController { + const controller = new AbortController() + this.controllers.set(runId, controller) + return controller + } + wrapDispatcher(inner: HeadlessAutomationDispatcher): HeadlessAutomationDispatcher { return async (request) => { - const controller = new AbortController() - this.controllers.set(request.run.id, controller) + const controller = this.controllers.get(request.run.id) + if (!controller || controller.signal !== request.completionSignal) { + throw new Error('Headless completion abort must begin before dispatch.') + } try { - const launch = await inner({ - ...request, - completionSignal: controller.signal - }) + const launch = await inner(request) this.trackCompletion(request.run.id, controller, launch) return { ...launch, completionAbortSignal: controller.signal } } catch (error) { @@ -41,10 +46,12 @@ export class HeadlessCompletionAbortRegistry { this.controllers.delete(runId) return } - void launch.completion.finally(() => { - if (this.controllers.get(runId) === controller) { - this.controllers.delete(runId) - } - }) + void launch.completion + .finally(() => { + if (this.controllers.get(runId) === controller) { + this.controllers.delete(runId) + } + }) + .catch(() => {}) } } diff --git a/src/main/automations/headless-dispatch-runner.ts b/src/main/automations/headless-dispatch-runner.ts index ba0911fe9190..558a239b8ee3 100644 --- a/src/main/automations/headless-dispatch-runner.ts +++ b/src/main/automations/headless-dispatch-runner.ts @@ -17,6 +17,7 @@ export type HeadlessAutomationDispatchContext = { run: AutomationRun target: Extract dispatcher: HeadlessAutomationDispatcher + beginHeadlessCompletionAbort: (runId: string) => AbortController runs: AutomationRunWriter runPrecheck: () => Promise markDispatchResult: (result: AutomationDispatchResult) => Promise @@ -53,7 +54,13 @@ export async function runHeadlessAutomationDispatch( }) } try { - const launch = await ctx.dispatcher({ automation, run, target }) + const completionAbort = ctx.beginHeadlessCompletionAbort(run.id) + const launch = await ctx.dispatcher({ + automation, + run, + target, + completionSignal: completionAbort.signal + }) const launchRunTarget = { workspaceId: launch.workspaceId, workspaceDisplayName: launch.workspaceDisplayName ?? null, diff --git a/src/main/automations/retained-run-reconciliation.test.ts b/src/main/automations/retained-run-reconciliation.test.ts index 57afd5ba963f..b83e80dc849e 100644 --- a/src/main/automations/retained-run-reconciliation.test.ts +++ b/src/main/automations/retained-run-reconciliation.test.ts @@ -9,6 +9,7 @@ import { type AutomationRun } from '../../shared/automations-types' import { AutomationService } from './service' +import type { HeadlessAutomationDispatcher } from './headless-dispatch' import type { AutomationRunCompletionObservation, AutomationRunTerminalObserver @@ -52,6 +53,8 @@ const LAUNCH_TARGET = { terminalPtyId: 'pty-1' } +const headlessLaunchOnly: HeadlessAutomationDispatcher = async () => ({ ...LAUNCH_TARGET }) + /** Well past the reconciler's post-ready settle window. */ const AFTER_SETTLE_MS = 10 * 60 * 1000 @@ -217,7 +220,7 @@ describe('reconciling retained runs against a graph that has not published yet', const retained = retainDispatchedRun(store, automation) const surface = createPaneSurface() const service = new AutomationService(store, { - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, terminalObserver: surface.observer }) diff --git a/src/main/automations/run-completion-watcher.test.ts b/src/main/automations/run-completion-watcher.test.ts index 3e694982f5ea..16a733154725 100644 --- a/src/main/automations/run-completion-watcher.test.ts +++ b/src/main/automations/run-completion-watcher.test.ts @@ -6,6 +6,7 @@ import type { Repo } from '../../shared/repo-types' import type { Automation, AutomationRun } from '../../shared/automations-types' import type { AutomationsChangedPayload } from '../../shared/runtime-client-events' import { AutomationService } from './service' +import type { HeadlessAutomationDispatcher } from './headless-dispatch' import type { AutomationRunCompletionObservation, AutomationRunTerminalObserver @@ -72,6 +73,8 @@ const LAUNCH_TARGET = { terminalPtyId: 'pty-1' } +const headlessLaunchOnly: HeadlessAutomationDispatcher = async () => ({ ...LAUNCH_TARGET }) + function readRun(store: TestStore, automationId: string, runId: string): AutomationRun { const run = store.listAutomationRuns(automationId).find((entry) => entry.id === runId) if (!run) { @@ -107,7 +110,7 @@ describe('authority-owned automation run completion', () => { const service = new AutomationService(store, { // Why: a dispatcher without a completion promise is exactly the case that // used to strand a run at 'dispatched' for the process lifetime. - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, terminalObserver: createObserver(async () => ({ status: 'completed', error: null })) }) @@ -123,7 +126,7 @@ describe('authority-owned automation run completion', () => { const store = await createStore() const automation = createAutomation(store) const service = new AutomationService(store, { - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, terminalObserver: createObserver( async () => ({ status: 'completed' }), () => null @@ -145,7 +148,7 @@ describe('authority-owned automation run completion', () => { const automation = createAutomation(store) const logged = vi.spyOn(console, 'error').mockImplementation(() => {}) const service = new AutomationService(store, { - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, terminalObserver: createObserver(async () => { throw new Error('terminal_handle_stale') }) @@ -239,7 +242,7 @@ describe('authority-owned automation run completion', () => { const automation = createAutomation(store) let resolveObservation: ((value: AutomationRunCompletionObservation) => void) | null = null const service = new AutomationService(store, { - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, terminalObserver: createObserver( () => new Promise((resolve) => { @@ -293,7 +296,8 @@ describe('authority-owned automation run completion', () => { const run = await service.runNow(automation.id) expect(readRun(store, automation.id, run.id).status).toBe('dispatched') service.stop() - expect(completionSignal?.aborted).toBe(true) + expect(completionSignal).not.toBeNull() + expect(completionSignal!.aborted).toBe(true) resolveCompletion({ status: 'completed', error: null }) await Promise.resolve() await Promise.resolve() @@ -305,7 +309,7 @@ describe('authority-owned automation run completion', () => { const automation = createAutomation(store) const aborted: boolean[] = [] const service = new AutomationService(store, { - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, terminalObserver: createObserver( (signal) => new Promise((_resolve, reject) => { @@ -331,7 +335,7 @@ describe('authority-owned automation run completion', () => { const automation = createAutomation(store) const publish = vi.fn() const service = new AutomationService(store, { - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, onAutomationsChanged: publish }) @@ -358,7 +362,7 @@ describe('automationsChanged publication', () => { const automation = createAutomation(store) const seen: { payload: AutomationsChangedPayload; status: string | undefined }[] = [] const service = new AutomationService(store, { - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, onAutomationsChanged: (payload) => { seen.push({ payload, @@ -394,7 +398,7 @@ describe('automationsChanged publication', () => { // An old client's event switch has no automationsChanged branch at all. const oldClientState = { repoRefreshes: 0 } const service = new AutomationService(store, { - headlessDispatcher: async () => ({ ...LAUNCH_TARGET }), + headlessDispatcher: headlessLaunchOnly, onAutomationsChanged: (payload) => { if ((payload as { type?: string }).type === 'reposChanged') { oldClientState.repoRefreshes += 1 diff --git a/src/main/automations/service-precheck.test.ts b/src/main/automations/service-precheck.test.ts index c13359b546d9..1527686e4435 100644 --- a/src/main/automations/service-precheck.test.ts +++ b/src/main/automations/service-precheck.test.ts @@ -232,6 +232,7 @@ describe('AutomationService prechecks', () => { run, target: { ok: true, cwd: '/repo', repo: store.getRepo('r1')! }, dispatcher: headlessDispatcher, + beginHeadlessCompletionAbort: () => new AbortController(), runs: createAutomationRunWriter(store, null), runPrecheck: () => service.runPrecheck(automation.id, run.id), markDispatchResult: (result) => service.markDispatchResult(result), diff --git a/src/main/automations/service.ts b/src/main/automations/service.ts index ee745bdc6744..5643415a6ed1 100644 --- a/src/main/automations/service.ts +++ b/src/main/automations/service.ts @@ -314,6 +314,7 @@ export class AutomationService { run, target, dispatcher: this.headlessDispatcher, + beginHeadlessCompletionAbort: (runId) => this.headlessCompletionAbort.beginRun(runId), runs: this.runs, runPrecheck: () => this.runPrecheck(automation.id, run.id), markDispatchResult: (result) => this.markDispatchResult(result), From e53ed09390fccab9a7b15e29e2378faeb3d5621c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 26 Sep 2026 00:11:25 +0000 Subject: [PATCH 8/8] test(automations): await dispatched status after wrapped headless dispatch --- src/main/automations/service.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/automations/service.test.ts b/src/main/automations/service.test.ts index 7077a70b6d10..11f2084aef7d 100644 --- a/src/main/automations/service.test.ts +++ b/src/main/automations/service.test.ts @@ -413,8 +413,11 @@ describe('AutomationService', () => { vi.setSystemTime(afterRunAt) service.start() await vi.waitFor(() => expect(headlessDispatcher).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => { + const pendingRun = store.listAutomationRuns(automation.id)[0] + expect(pendingRun?.status).toBe('dispatched') + }) const run = store.listAutomationRuns(automation.id)[0] - expect(run?.status).toBe('dispatched') expect(run?.scheduledFor).toBe(scheduledRunAt) expect(headlessDispatcher).toHaveBeenCalledWith( expect.objectContaining({ automation: expect.objectContaining({ id: automation.id }) })