Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 118 additions & 0 deletions src/main/automations/headless-run-completion.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/**
* When a headlessly dispatched run is complete. A ready shell prompt satisfies `tui-idle` too,
* so a prompt typed at a shell whose agent is not installed ("command not found") read as a
* finished run. The agent's own status for the run's pane, reported after dispatch, completes it
* as on the desktop. Not every agent reports status (no hooks on the host, no recognised title),
* so for those an idle pane still means done, but only once the agent had time to start and the
* pane shows no shell refusing its command.
*/
import type { AutomationRunCompletionObservation } from './run-completion-watcher'
import { createHeadlessAutomationOutputSnapshotBuffer } from './headless-dispatch'

/** Covers agent spin-up over SSH before an idle pane without agent status is believed. */
export const HEADLESS_AGENT_START_GRACE_MS = 2 * 60 * 1000
const AGENT_START_POLL_MS = 1_000
const TERMINAL_SNAPSHOT_LIMIT = 2_000
/** Only output after the prompt: a refusal further up predates this dispatch. */
const MISSING_COMMAND_TAIL_LINES = 6

// bash, zsh, dash/sh, fish, PowerShell and cmd.exe refusing a command that does not exist.
const MISSING_COMMAND_PATTERNS = [
/command not found/i,
/^\S+: \d+: \S+: not found$/i,
/unknown command/i,
/is not recognized as (?:the name of a cmdlet|an internal or external command)/i
]

/** A shell refusing one of the agent's own commands; an agent's tool output never matches. */
export function findMissingCommandLine(
tail: readonly string[],
commands: readonly string[]
): string | null {
const names = commands.map((command) => command.split(/\s+/)[0]).filter(Boolean)
const recent = tail
.map((line) => line.trim())
.filter(Boolean)
.slice(-MISSING_COMMAND_TAIL_LINES)
return (
recent.find(
(line) =>
MISSING_COMMAND_PATTERNS.some((pattern) => pattern.test(line)) &&
names.some((name) => line.includes(name))
) ?? null
)
}

export type HeadlessRunCompletionHost = {
waitForTerminal(
handle: string,
options?: { condition?: 'tui-idle' }
): Promise<{ satisfied: boolean; blockedReason?: string }>
readTerminal(handle: string, opts?: { limit?: number }): Promise<{ tail: string[] }>
/** Agent status rows for a pane, from hooks, OSC and titles alike. */
getAgentStatusRowsForPane(paneKey: string): readonly { receivedAt: number }[]
}

export async function observeHeadlessRunCompletion(
host: HeadlessRunCompletionHost,
run: {
handle: string
paneKey: string | null
dispatchedAt: number
/** The names the agent's command may run under, to tell its refusal from its output. */
agentCommands: readonly string[]
},
clock: { now: () => number; sleep: (ms: number) => Promise<void> } = {
now: Date.now,
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms))
}
): Promise<AutomationRunCompletionObservation> {
const agentReported = (): boolean =>
run.paneKey !== null &&
host.getAgentStatusRowsForPane(run.paneKey).some((row) => row.receivedAt >= run.dispatchedAt)
for (;;) {
const wait = await host.waitForTerminal(run.handle, { condition: 'tui-idle' })
const tail = await readTail(host, run.handle)
const outputSnapshot = snapshotOf(tail)
if (!wait.satisfied) {
return {
status: 'dispatch_failed',
outputSnapshot,
error: wait.blockedReason
? `Automation agent is blocked: ${wait.blockedReason}.`
: 'Automation agent did not report completion.'
}
}
if (agentReported()) {
return { status: 'completed', outputSnapshot, error: null }
}
const missing = findMissingCommandLine(tail, run.agentCommands)
if (missing) {
return {
status: 'dispatch_failed',
outputSnapshot,
error: `Automation agent did not start; this host could not run its command (${missing}).`
}
}
// An agent that never reports status keeps the idle-means-done rule, after its start window.
if (clock.now() - run.dispatchedAt >= HEADLESS_AGENT_START_GRACE_MS) {
return { status: 'completed', outputSnapshot, error: null }
}
await clock.sleep(AGENT_START_POLL_MS)
}
}

async function readTail(host: HeadlessRunCompletionHost, handle: string): Promise<string[]> {
try {
return (await host.readTerminal(handle, { limit: TERMINAL_SNAPSHOT_LIMIT })).tail
} catch {
// The terminal can exit between the wait and the read; a missing tail is not a failure.
return []
}
}

function snapshotOf(tail: readonly string[]): AutomationRunCompletionObservation['outputSnapshot'] {
const buffer = createHeadlessAutomationOutputSnapshotBuffer()
buffer.append(tail.join('\n'))
return buffer.snapshot()
}
129 changes: 129 additions & 0 deletions src/main/automations/runtime-automation-service.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { HeadlessAutomationDispatcher } from './headless-dispatch'

const captured = vi.hoisted((): { dispatcher: unknown } => ({ dispatcher: null }))

vi.mock('./service', () => ({
AutomationService: class {
constructor(_store: unknown, opts: { headlessDispatcher?: unknown }) {
captured.dispatcher = opts.headlessDispatcher
}
}
}))

import { createRuntimeAutomationService } from './runtime-automation-service'
import { HEADLESS_AGENT_START_GRACE_MS } from './headless-run-completion'

const PANE_KEY = 'tab-1:pane-1'

const GOOSE_MISSING = ['$ goose run', 'bash: goose: command not found', '$']

function headlessRuntime(rows: () => { receivedAt: number }[], tail: string[] = GOOSE_MISSING) {
return {
setAutomationService: vi.fn(),
notifyAutomationsChanged: vi.fn(),
launchAgentTerminal: vi.fn(async () => ({
handle: 'terminal-1',
tabId: 'tab-1',
paneKey: PANE_KEY,
ptyId: 'pty-1',
worktreeId: 'wt-1'
})),
showManagedWorktree: vi.fn(async () => ({ displayName: 'repo' })),
// A ready shell prompt satisfies tui-idle whether or not an agent ever ran.
waitForTerminal: vi.fn(async () => ({ satisfied: true })),
readTerminal: vi.fn(async () => ({ tail })),
getAgentStatusRowsForPane: vi.fn(rows)
}
}

function dispatch(runtime: ReturnType<typeof headlessRuntime>) {
createRuntimeAutomationService({
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mocked service never reads the store.
store: {} as never,
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the dispatcher reads only the members faked above.
runtime: runtime as never,
headless: true
})
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: captured from the mocked constructor above.
const dispatcher = captured.dispatcher as HeadlessAutomationDispatcher
return dispatchOnce(dispatcher)
}

async function dispatchOnce(dispatcher: HeadlessAutomationDispatcher) {
const launch = await dispatcher({
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a reuse-workspace automation; only these fields are read.
automation: {
workspaceMode: 'existing',
workspaceId: 'wt-1',
agentId: 'goose',
prompt: 'hi'
} as never,
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the title is read.
run: { title: 'Nightly' } as never,
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: unused for an existing workspace.
target: {} as never
})
if (!launch.completion) {
throw new Error('a headless dispatch always reports its completion')
}
return { ...launch, completion: launch.completion }
}

afterEach(() => {
vi.useRealTimers()
})

describe('headless automation run completion', () => {
it.each([
['bash', 'bash: goose: command not found'],
['zsh', 'zsh: command not found: goose'],
['dash', 'sh: 1: goose: not found'],
['fish', 'fish: Unknown command: goose'],
['PowerShell', "goose: The term 'goose' is not recognized as the name of a cmdlet"]
])('fails, never completes, a run whose agent %s cannot find', async (_shell, refusal) => {
const runtime = headlessRuntime(() => [], ['$ goose run', refusal, '$'])
const observation = await (await dispatch(runtime)).completion
expect(observation.status).toBe('dispatch_failed')
expect(observation.error).toContain(refusal)
})

it('completes once the agent itself reported for the run pane after dispatch', async () => {
const runtime = headlessRuntime(() => [{ receivedAt: Date.now() }], ['done', '$'])
const result = await dispatch(runtime)
await expect(result.completion).resolves.toMatchObject({ status: 'completed', error: null })
expect(runtime.getAgentStatusRowsForPane).toHaveBeenCalledWith(PANE_KEY)
})

it('keeps idle-means-done for an agent that never reports status, after its start window', async () => {
vi.useFakeTimers()
const runtime = headlessRuntime(() => [{ receivedAt: 1 }], ['summary written', '$'])
const result = await dispatch(runtime)
const settled = vi.fn()
void result.completion.then(settled)
await vi.advanceTimersByTimeAsync(10_000)
expect(settled).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(HEADLESS_AGENT_START_GRACE_MS)
await expect(result.completion).resolves.toMatchObject({ status: 'completed' })
})

it("never reads an agent's own tool output as its command missing", async () => {
vi.useFakeTimers()
const tail = ['running tests', 'bash: pytest: command not found', 'fell back to unittest', '$']
const runtime = headlessRuntime(() => [], tail)
const result = await dispatch(runtime)
await vi.advanceTimersByTimeAsync(HEADLESS_AGENT_START_GRACE_MS + 1_000)
await expect(result.completion).resolves.toMatchObject({ status: 'completed' })
})

it('waits for an agent that reports after the shell first looked idle', async () => {
vi.useFakeTimers()
let rows: { receivedAt: number }[] = []
const runtime = headlessRuntime(() => rows, ['$'])
const result = await dispatch(runtime)
await vi.advanceTimersByTimeAsync(5_000)
rows = [{ receivedAt: Date.now() }]
await vi.advanceTimersByTimeAsync(1_000)
await expect(result.completion).resolves.toMatchObject({ status: 'completed' })
})
})
45 changes: 21 additions & 24 deletions src/main/automations/runtime-automation-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@ import type { CodexUsageStore } from '../codex-usage/store'
import type { Store } from '../persistence'
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import { AutomationService } from './service'
import { createHeadlessAutomationOutputSnapshotBuffer } from './headless-dispatch'
import { observeHeadlessRunCompletion } from './headless-run-completion'
import {
getTuiAgentDetectCommands,
isTuiAgent,
TUI_AGENT_CONFIG
} from '../../shared/tui-agent-config'
import { buildHeadlessAutomationWorktreeCreateArgs } from './headless-workspace-create'
import { createRuntimeAutomationRunTerminalObserver } from './runtime-terminal-run-observer'

Expand All @@ -28,7 +33,7 @@ export function createRuntimeAutomationService(input: {
allowRemoteHostScheduling: input.headless,
headlessDispatcher: input.headless
? async ({ automation, run, target }) => {
const terminalSnapshotLimit = 2_000
const dispatchedAt = Date.now()
let terminalHandle: string
let terminalSessionId: string | null = null
let terminalPaneKey: string | null = null
Expand Down Expand Up @@ -68,28 +73,12 @@ export function createRuntimeAutomationService(input: {
const worktree = await runtime.showManagedWorktree(`id:${workspaceId}`)
workspaceDisplayName = worktree.displayName ?? null
}
const completion = (async () => {
const wait = await runtime.waitForTerminal(terminalHandle, { condition: 'tui-idle' })
const read = await runtime.readTerminal(terminalHandle, {
limit: terminalSnapshotLimit
})
const snapshotBuffer = createHeadlessAutomationOutputSnapshotBuffer()
snapshotBuffer.append(read.tail.join('\n'))
if (wait.satisfied) {
return {
status: 'completed' as const,
outputSnapshot: snapshotBuffer.snapshot(),
error: null
}
}
return {
status: 'dispatch_failed' as const,
outputSnapshot: snapshotBuffer.snapshot(),
error: wait.blockedReason
? `Automation agent is blocked: ${wait.blockedReason}.`
: 'Automation agent did not report completion.'
}
})()
const completion = observeHeadlessRunCompletion(runtime, {
handle: terminalHandle,
paneKey: terminalPaneKey,
dispatchedAt,
agentCommands: automationAgentCommands(automation.agentId)
})
return {
workspaceId,
workspaceDisplayName,
Expand All @@ -104,3 +93,11 @@ export function createRuntimeAutomationService(input: {
runtime.setAutomationService(service)
return service
}

function automationAgentCommands(agentId: string): string[] {
if (!isTuiAgent(agentId)) {
return []
}
const config = TUI_AGENT_CONFIG[agentId]
return [...getTuiAgentDetectCommands(config), config.launchCmd]
}
5 changes: 5 additions & 0 deletions src/main/runtime/orca-runtime-preserved-branch-cleanup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ export class OrcaRuntimeWithPreservedBranchCleanup extends OrcaRuntimeWithTermin
| ((paneKey: string) => AgentStatusIpcPayload[])
| null

/** Agent status rows this host holds for a pane, from hooks, OSC and titles alike. */
getAgentStatusRowsForPane(paneKey: string): AgentStatusIpcPayload[] {
return this.getAgentProviderSessionRowsForPaneFn?.(paneKey) ?? []
}

protected readonly attestAgentHookCompatibilityAuthorityFn:
| ((candidate: {
paneKey: string
Expand Down
Loading