Skip to content
Closed
57 changes: 57 additions & 0 deletions src/main/automations/headless-completion-abort.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import type {
HeadlessAutomationDispatcher,
HeadlessAutomationDispatchLaunch
} from './headless-dispatch'

/** Tracks headless completion observers so AutomationService.stop() can abort them. */
export class HeadlessCompletionAbortRegistry {
private readonly controllers = new Map<string, AbortController>()

beginRun(runId: string): AbortController {
const controller = new AbortController()
this.controllers.set(runId, controller)
return controller
}

wrapDispatcher(inner: HeadlessAutomationDispatcher): HeadlessAutomationDispatcher {
return async (request) => {
const controller = this.controllers.get(request.run.id)
if (!controller || controller.signal !== request.completionSignal) {
throw new Error('Headless completion abort must begin before dispatch.')
}
try {
const launch = await inner(request)
this.trackCompletion(request.run.id, controller, launch)
return { ...launch, completionAbortSignal: controller.signal }
} catch (error) {
this.controllers.delete(request.run.id)
throw error
}
}
}

abortAll(): void {
for (const controller of this.controllers.values()) {
controller.abort()
}
this.controllers.clear()
}

private trackCompletion(
runId: string,
controller: AbortController,
launch: HeadlessAutomationDispatchLaunch
): void {
if (!launch.completion) {
this.controllers.delete(runId)
return
}
void launch.completion
.finally(() => {
if (this.controllers.get(runId) === controller) {
this.controllers.delete(runId)
}
})
.catch(() => {})
}
}
38 changes: 31 additions & 7 deletions src/main/automations/headless-dispatch-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export type HeadlessAutomationDispatchContext = {
run: AutomationRun
target: Extract<AutomationRunTargetResult, { ok: true }>
dispatcher: HeadlessAutomationDispatcher
beginHeadlessCompletionAbort: (runId: string) => AbortController
runs: AutomationRunWriter
runPrecheck: () => Promise<AutomationPrecheckResult | null>
markDispatchResult: (result: AutomationDispatchResult) => Promise<AutomationRun>
Expand All @@ -27,6 +28,16 @@ function describeDispatchError(error: unknown): string {
return error instanceof Error ? error.message : String(error)
}

function shouldIgnoreHeadlessCompletion(
error: unknown,
abortSignal: AbortSignal | undefined
): boolean {
if (abortSignal?.aborted) {
return true
}
return error instanceof Error && error.message === 'request_aborted'
}

export async function runHeadlessAutomationDispatch(
ctx: HeadlessAutomationDispatchContext
): Promise<AutomationRun> {
Expand All @@ -43,7 +54,13 @@ export async function runHeadlessAutomationDispatch(
})
}
try {
const launch = await ctx.dispatcher({ automation, run, target })
const completionAbort = ctx.beginHeadlessCompletionAbort(run.id)
const launch = await ctx.dispatcher({
automation,
run,
target,
completionSignal: completionAbort.signal
})
const launchRunTarget = {
workspaceId: launch.workspaceId,
workspaceDisplayName: launch.workspaceDisplayName ?? null,
Expand All @@ -63,25 +80,32 @@ export async function runHeadlessAutomationDispatch(
ctx.watchRun(updated)
return updated
}
const completionAbortSignal = launch.completionAbortSignal
void launch.completion
.then((completion) =>
ctx.markDispatchResult({
.then((completion) => {
if (completionAbortSignal?.aborted) {
return
}
return ctx.markDispatchResult({
runId: run.id,
status: completion.status,
...launchRunTarget,
precheckResult,
outputSnapshot: completion.outputSnapshot ?? null,
error: completion.error ?? null
})
)
.catch((error) =>
ctx.markDispatchResult({
})
.catch((error) => {
if (shouldIgnoreHeadlessCompletion(error, completionAbortSignal)) {
return
}
return ctx.markDispatchResult({
runId: run.id,
status: 'dispatch_failed',
...launchRunTarget,
error: describeDispatchError(error)
})
)
})
return updated
} catch (error) {
return runs.updateRun({
Expand Down
136 changes: 136 additions & 0 deletions src/main/automations/headless-dispatch-terminal-completion.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createHeadlessAutomationCompletion } from './headless-dispatch-terminal-completion'
import {
createRuntimeAutomationRunTerminalObserver,
type AutomationRunTerminalHost
} from './runtime-terminal-run-observer'

const HANDLE = 'terminal-1'
const RUNTIME_TUI_IDLE_TIMEOUT_MS = 5 * 60 * 1000

type FakePane = {
lastAgentStatus: 'idle' | 'working' | 'permission' | null
}

type FakeWaitResult = {
handle: string
condition: 'tui-idle'
satisfied: boolean
status: 'running' | 'exited' | 'unknown'
exitCode: number | null
}

type FakeWaiter = {
resolve: (value: FakeWaitResult) => void
reject: (error: Error) => void
timer: ReturnType<typeof setTimeout>
}

function createFakeRuntime(initial: Partial<FakePane>) {
const pane: FakePane = {
lastAgentStatus: null,
...initial
}
const waiters = new Set<FakeWaiter>()

const satisfiedNow = (): boolean => pane.lastAgentStatus === 'idle'

const runtime: AutomationRunTerminalHost & {
setPane: (next: Partial<FakePane>) => void
} = {
getTerminalHandleForPaneKey: () => HANDLE,
readTerminal: async () => ({ tail: ['agent finished after a long command'] }),
waitForTerminal: (_handle, options) => {
if (options?.signal?.aborted) {
return Promise.reject(new Error('request_aborted'))
}
if (satisfiedNow()) {
return Promise.resolve({
handle: HANDLE,
condition: 'tui-idle',
satisfied: true,
status: 'running',
exitCode: null
})
}
return new Promise((resolve, reject) => {
const waiter: FakeWaiter = {
resolve,
reject,
timer: setTimeout(() => {
waiters.delete(waiter)
reject(new Error('timeout'))
}, options?.timeoutMs ?? RUNTIME_TUI_IDLE_TIMEOUT_MS)
}
waiters.add(waiter)
})
},
setPane: (next) => {
Object.assign(pane, next)
if (!satisfiedNow()) {
return
}
for (const waiter of waiters) {
waiters.delete(waiter)
clearTimeout(waiter.timer)
waiter.resolve({
handle: HANDLE,
condition: 'tui-idle',
satisfied: true,
status: 'running',
exitCode: null
})
}
}
}
return runtime
}

describe('createHeadlessAutomationCompletion (#22725)', () => {
beforeEach(() => {
vi.useFakeTimers()
})

afterEach(() => {
vi.useRealTimers()
})

it('routes headless completion through observeCompletion', async () => {
const observeCompletion = vi.fn(async () => ({
status: 'completed' as const,
outputSnapshot: null,
error: null
}))
await expect(
createHeadlessAutomationCompletion({
observeCompletion,
terminalHandle: HANDLE,
signal: new AbortController().signal
})
).resolves.toMatchObject({ status: 'completed', error: null })
expect(observeCompletion).toHaveBeenCalledWith(
HANDLE,
expect.objectContaining({ signal: expect.any(AbortSignal) })
)
})

it('still completes after the agent works past the 5-minute tui-idle default', async () => {
const runtime = createFakeRuntime({ lastAgentStatus: 'working' })
const observer = createRuntimeAutomationRunTerminalObserver(runtime)
const completion = createHeadlessAutomationCompletion({
observeCompletion: (handle, options) => observer.observeCompletion(handle, options),
terminalHandle: HANDLE,
signal: new AbortController().signal
})

// Agent keeps working past the runtime default that used to reject headless.
await vi.advanceTimersByTimeAsync(RUNTIME_TUI_IDLE_TIMEOUT_MS + 18_000)
runtime.setPane({ lastAgentStatus: 'idle' })
await vi.advanceTimersByTimeAsync(10)

await expect(completion).resolves.toMatchObject({
status: 'completed',
error: null
})
})
})
21 changes: 21 additions & 0 deletions src/main/automations/headless-dispatch-terminal-completion.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import type {
AutomationRunCompletionObservation,
AutomationRunTerminalObserver
} from './run-completion-watcher'

/**
* Headless dispatch used to call `waitForTerminal({ condition: 'tui-idle' })`
* with no `timeoutMs`, so it inherited the runtime's 5-minute default and marked
* longer agent runs as `dispatch_failed` / `timeout` while the agent was still
* working (#22725). Route through the same observer retained runs use so each
* tui-idle timeout re-arms until the shared 6h deadline.
*/
export function createHeadlessAutomationCompletion(args: {
observeCompletion: AutomationRunTerminalObserver['observeCompletion']
terminalHandle: string
signal: AbortSignal
}): Promise<AutomationRunCompletionObservation> {
return args.observeCompletion(args.terminalHandle, {
signal: args.signal
})
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
3 changes: 3 additions & 0 deletions src/main/automations/headless-dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,15 @@ export type HeadlessAutomationDispatchLaunch = {
outputSnapshot?: AutomationRunOutputSnapshot | null
error?: string | null
}>
/** Set by AutomationService so headless completion can be dropped after stop(). */
completionAbortSignal?: AbortSignal
}

export type HeadlessAutomationDispatcher = (request: {
automation: Automation
run: AutomationRun
target: Extract<AutomationRunTargetResult, { ok: true }>
completionSignal: AbortSignal
Comment thread
pullfrog[bot] marked this conversation as resolved.
Resolved
}) => Promise<HeadlessAutomationDispatchLaunch>

export function createHeadlessAutomationOutputSnapshotBuffer(): {
Expand Down
5 changes: 4 additions & 1 deletion src/main/automations/retained-run-reconciliation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
type AutomationRun
} from '../../shared/automations-types'
import { AutomationService } from './service'
import type { HeadlessAutomationDispatcher } from './headless-dispatch'
import type {
AutomationRunCompletionObservation,
AutomationRunTerminalObserver
Expand Down Expand Up @@ -52,6 +53,8 @@ const LAUNCH_TARGET = {
terminalPtyId: 'pty-1'
}

const headlessLaunchOnly: HeadlessAutomationDispatcher = async () => ({ ...LAUNCH_TARGET })

/** Well past the reconciler's post-ready settle window. */
const AFTER_SETTLE_MS = 10 * 60 * 1000

Expand Down Expand Up @@ -217,7 +220,7 @@ describe('reconciling retained runs against a graph that has not published yet',
const retained = retainDispatchedRun(store, automation)
const surface = createPaneSurface()
const service = new AutomationService(store, {
headlessDispatcher: async () => ({ ...LAUNCH_TARGET }),
headlessDispatcher: headlessLaunchOnly,
terminalObserver: surface.observer
})

Expand Down
Loading