From a8f3368a6081ae24089540a7890d07083081d6cf Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 14:43:14 -0800 Subject: [PATCH 01/21] Stop retrying final native agent startup refusals --- .../acp/acp-session-reopen-failure.test.ts | 39 +++++ src/main/acp/acp-session-reopen-failure.ts | 4 +- src/main/acp/acp-startup-failure.ts | 12 ++ src/main/acp/acp-structured-handshake.ts | 1 - .../claude-agent-sdk-control-requests.ts | 11 +- .../claude/claude-agent-sdk-process-spawn.ts | 4 +- .../claude-startup-answer-observation.ts | 43 +++++ src/main/claude/claude-startup-failure.ts | 46 ++++++ .../claude/claude-stream-json-connection.ts | 15 +- .../fixtures/startup-sdk-process-exit.json | 22 +++ .../claude-structured-session-acquisition.ts | 7 +- ...laude-structured-session-exit-lifecycle.ts | 31 ++-- ...structured-session-startup-failure.test.ts | 139 ++++++++++++++++ .../claude-structured-session-startup.ts | 69 ++++++-- .../codex-app-server-record-dispatch.ts | 3 +- .../codex-app-server-request-error.ts | 5 +- .../codex/app-server/codex-startup-failure.ts | 46 ++++++ ...art-resume-canonical-filename-refusal.json | 6 + .../codex-scripted-child.test-fixture.ts | 12 ++ .../structured-agent-session-provider-open.ts | 22 ++- ...structured-agent-session-start-attempts.ts | 57 ++++++- ...agent-session-start-retry-recorded.test.ts | 154 ++++++++++++++++-- .../structured-agent-session-start-retry.ts | 48 +++--- ...gent-session-start-settlement-host.test.ts | 8 +- ...ssion-startup-attempt-host-test-support.ts | 31 +++- ...agent-session-startup-attempt-host.test.ts | 84 ++++++++++ ...ured-agent-session-startup-attempt-mint.ts | 39 +++++ ...tructured-agent-session-startup-attempt.ts | 63 +++---- src/main/pi/pi-scripted-child.test-fixture.ts | 7 + ...gent-session-adapter-start-failure.test.ts | 38 ++++- ...tured-agent-scripted-child.test-fixture.ts | 2 + 31 files changed, 932 insertions(+), 136 deletions(-) create mode 100644 src/main/acp/acp-session-reopen-failure.test.ts create mode 100644 src/main/acp/acp-startup-failure.ts create mode 100644 src/main/claude/claude-startup-answer-observation.ts create mode 100644 src/main/claude/claude-startup-failure.ts create mode 100644 src/main/claude/fixtures/startup-sdk-process-exit.json create mode 100644 src/main/claude/structured/claude-structured-session-startup-failure.test.ts create mode 100644 src/main/codex/app-server/codex-startup-failure.ts create mode 100644 src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-mint.ts diff --git a/src/main/acp/acp-session-reopen-failure.test.ts b/src/main/acp/acp-session-reopen-failure.test.ts new file mode 100644 index 000000000000..c86573169515 --- /dev/null +++ b/src/main/acp/acp-session-reopen-failure.test.ts @@ -0,0 +1,39 @@ +import { openAcpAdapterRig } from './acp-structured-adapter.test-support' +import { closeProviderTimelineRigs } from '../native-chat/agent-session-timeline/provider-timeline-assembler-test-support' +import { afterEach, expect, it, vi } from 'vitest' +import { AcpAgentError } from './acp-errors' +import { acpReopenTakeover } from './acp-session-reopen-failure' + +it.each(['ServerUnavailable', 'CatalogNotReady'])( + 'retains the saved session on a temporary %s load answer', + (errorName) => { + const error = new AcpAgentError(-32603, 'not ready', { errorName }) + const warn = vi.fn() + expect(() => + acpReopenTakeover( + error, + { key: 'saved-key', sessionId: 'saved-id', mayBeUnsaved: () => false }, + { over: false, now: 1, warn } + ) + ).toThrow(error) + expect(warn).not.toHaveBeenCalled() + } +) + +afterEach(closeProviderTimelineRigs) + +it('propagates a temporary load envelope before takeover or a dead-child assertion can replace it', async () => { + const rig = await openAcpAdapterRig({ + launch: { resume: { key: 'saved-key', sessionId: 'saved-id', mayBeUnsaved: () => false } }, + script: (agent) => + agent.on('session/load', (frame) => + agent.fail(frame, -32603, 'server unavailable', { errorName: 'ServerUnavailable' }) + ) + }) + await rig.acquire() + expect(rig.openFailure()).toBeInstanceOf(AcpAgentError) + expect(rig.openFailure()).toMatchObject({ data: { errorName: 'ServerUnavailable' } }) + expect(rig.sent('session/new')).toEqual([]) + expect(rig.sent('session/load')[0].params).toMatchObject({ sessionId: 'saved-id' }) + await rig.adapter.closeAll() +}) diff --git a/src/main/acp/acp-session-reopen-failure.ts b/src/main/acp/acp-session-reopen-failure.ts index f24ae0036251..3da135aa3f01 100644 --- a/src/main/acp/acp-session-reopen-failure.ts +++ b/src/main/acp/acp-session-reopen-failure.ts @@ -5,6 +5,8 @@ import type { ProviderTimelineEvent } from '../native-chat/agent-session-timelin import { AcpAuthRequiredError, AcpRpcError } from './acp-errors' import type { AcpStructuredLaunch } from './acp-structured-launch-resolution' +import { temporaryAcpStartupAnswer } from './acp-startup-failure' + const ACP_RESOURCE_NOT_FOUND = -32002 export const ACP_REOPEN_FAILED = 'ACP agent could not reopen its saved session; starting a new one' @@ -14,7 +16,7 @@ export function acpReopenTakeover( start: { over: boolean; now: number; warn: (fields: { scope: string; error: unknown }) => void } ): Pick { // A replacement session cannot resolve sign-in failure or an abandoned start. - if (error instanceof AcpAuthRequiredError || start.over) { + if (error instanceof AcpAuthRequiredError || temporaryAcpStartupAnswer(error) || start.over) { throw error } if ( diff --git a/src/main/acp/acp-startup-failure.ts b/src/main/acp/acp-startup-failure.ts new file mode 100644 index 000000000000..1b1d71a03910 --- /dev/null +++ b/src/main/acp/acp-startup-failure.ts @@ -0,0 +1,12 @@ +import { AcpAgentError } from './acp-errors' + +export function temporaryAcpStartupAnswer(error: unknown): boolean { + return ( + error instanceof AcpAgentError && + error.code === -32603 && + typeof error.data === 'object' && + error.data !== null && + 'errorName' in error.data && + (error.data.errorName === 'ServerUnavailable' || error.data.errorName === 'CatalogNotReady') + ) +} diff --git a/src/main/acp/acp-structured-handshake.ts b/src/main/acp/acp-structured-handshake.ts index 1a1a1bdd7b2d..240415542a62 100644 --- a/src/main/acp/acp-structured-handshake.ts +++ b/src/main/acp/acp-structured-handshake.ts @@ -86,7 +86,6 @@ export async function initializeAcpStructuredSession( replayed = input.lanes.finishLoad() attaching.translator.finishLoad() } catch (error) { - assertStarting() takeover = acpReopenTakeover(error, resume, { over: connection.closed || input.abandoned(), now: now(), diff --git a/src/main/claude/claude-agent-sdk-control-requests.ts b/src/main/claude/claude-agent-sdk-control-requests.ts index 066467060982..980093a4e9c3 100644 --- a/src/main/claude/claude-agent-sdk-control-requests.ts +++ b/src/main/claude/claude-agent-sdk-control-requests.ts @@ -7,9 +7,10 @@ import type { export class ClaudeControlRequestError extends Error { constructor( readonly subtype: string, - message: string + message: string, + options?: ErrorOptions ) { - super(message) + super(message, options) this.name = 'ClaudeControlRequestError' } } @@ -27,6 +28,10 @@ export class ClaudeControlRequestTimeoutError extends Error { // Exact SDK error used to distinguish query closure from CLI rejection. const QUERY_CLOSED_MESSAGE = 'Query closed before response received' +export function isClaudeQueryClosed(error: unknown): boolean { + return error instanceof Error && error.message === QUERY_CLOSED_MESSAGE +} + // The SDK implements getSettings but omits it from Query's declaration. type ClaudeQuerySettingsReader = { getSettings?: () => Promise } @@ -72,7 +77,7 @@ export function runClaudeControl( if (error instanceof ClaudeControlRequestError || message === QUERY_CLOSED_MESSAGE) { throw error } - throw new ClaudeControlRequestError(subtype, message) + throw new ClaudeControlRequestError(subtype, message, { cause: error }) }) if (timeoutMs === null) { return request diff --git a/src/main/claude/claude-agent-sdk-process-spawn.ts b/src/main/claude/claude-agent-sdk-process-spawn.ts index 34c9f80856c1..57a24b5f16f9 100644 --- a/src/main/claude/claude-agent-sdk-process-spawn.ts +++ b/src/main/claude/claude-agent-sdk-process-spawn.ts @@ -33,7 +33,8 @@ function definedEnv(env: Record): Record void ): ClaudeCodeProcessSpawn { let managed: ManagedProviderProcess | null = null return { @@ -59,6 +60,7 @@ export function createClaudeCodeProcessSpawn( } ) // The SDK drains stderr only for its own local spawn; the managed process drains it here. + observeChild?.(managed.child) return managed.child }, get child() { diff --git a/src/main/claude/claude-startup-answer-observation.ts b/src/main/claude/claude-startup-answer-observation.ts new file mode 100644 index 000000000000..efda9dd91b0a --- /dev/null +++ b/src/main/claude/claude-startup-answer-observation.ts @@ -0,0 +1,43 @@ +import { StringDecoder } from 'node:string_decoder' +import type { PipedChildProcess } from '@orca/process-host/process-spec' +import { createIncrementalNdjsonFramer } from '../../shared/main-process-ndjson-framer' + +/** Observe before SDK cleanup can wait on transcript flushing. */ +export function claudeStartupAnswerObservation() { + let failure: Error | undefined + let detach = (): void => {} + const observe = (record: unknown): void => { + if (typeof record !== 'object' || record === null) { + return + } + if ( + 'type' in record && + record.type === 'result' && + 'is_error' in record && + record.is_error === true && + !('parent_tool_use_id' in record && record.parent_tool_use_id) + ) { + failure ??= Object.assign(new Error('claude reported a failed result during startup'), { + errorClass: 'error_result' + }) + } + } + return { + failure: () => failure, + finish: () => detach(), + attach: (child: PipedChildProcess): void => { + const decoder = new StringDecoder('utf8') + const framer = createIncrementalNdjsonFramer(observe, () => {}, { + maxLineBytes: Number.POSITIVE_INFINITY + }) + const onData = (chunk: Buffer | string): void => + framer.feed(typeof chunk === 'string' ? chunk : decoder.write(chunk)) + detach = () => { + child.stdout.removeListener('data', onData) + framer.reset() + } + child.stdout.on('data', onData) + child.stdout.once('close', () => detach()) + } + } +} diff --git a/src/main/claude/claude-startup-failure.ts b/src/main/claude/claude-startup-failure.ts new file mode 100644 index 000000000000..e425ed63ca90 --- /dev/null +++ b/src/main/claude/claude-startup-failure.ts @@ -0,0 +1,46 @@ +import { findInErrorChain } from '../provider-process/provider-error-chain' +import { ClaudeControlRequestError } from './claude-agent-sdk-control-requests' + +/** SDK runtime tags are not in its public Error declaration. */ +export function claudeStartupErrorClass(error: Error): string | undefined { + return 'errorClass' in error && typeof error.errorClass === 'string' + ? error.errorClass + : undefined +} + +export function claudeStartupAnswer(error: Error): 'temporary' | 'final' | undefined { + const tag = claudeStartupErrorClass(error) + if (tag === 'process_exited_nonzero' || tag === 'process_killed_by_signal') { + return 'temporary' + } + if (tag === 'executable_launch_failed' || tag === 'spawn_failed' || tag === 'process_error') { + return resourcePressure(error) ? 'temporary' : 'final' + } + if (tag !== undefined) { + return 'final' + } + if (error instanceof ClaudeControlRequestError) { + const origin = findInErrorChain(error.cause, (cause) => claudeStartupErrorClass(cause)) + if ( + origin === 'executable_launch_failed' || + origin === 'spawn_failed' || + origin === 'process_error' + ) { + return resourcePressure(error) ? 'temporary' : 'final' + } + return origin === 'process_exited_nonzero' || origin === 'process_killed_by_signal' + ? 'temporary' + : 'final' + } + return undefined +} + +function resourcePressure(error: Error): boolean { + return ( + findInErrorChain(error, (cause) => + 'code' in cause && ['EAGAIN', 'EMFILE', 'ENFILE'].some((code) => cause.code === code) + ? true + : undefined + ) === true + ) +} diff --git a/src/main/claude/claude-stream-json-connection.ts b/src/main/claude/claude-stream-json-connection.ts index 57397928f5eb..195548c29e7e 100644 --- a/src/main/claude/claude-stream-json-connection.ts +++ b/src/main/claude/claude-stream-json-connection.ts @@ -1,3 +1,4 @@ +import { claudeStartupAnswerObservation } from './claude-startup-answer-observation' import type { PipedProcessSpawner } from '@orca/process-host/process-spec' import type * as ClaudeAgentSdk from '@anthropic-ai/claude-agent-sdk' import type { CanUseTool, OnUserDialog, SDKUserMessage } from '@anthropic-ai/claude-agent-sdk' @@ -85,6 +86,8 @@ export type ClaudeStreamJsonConnection = ClaudeControlSurface & { /** What the ladder has observed so far; read after a `close()` that returned false. */ readonly exitVerdict: ClaudeChildExitVerdict readonly executableMissing?: boolean + exited?: Promise + startupFailure?: () => Error | undefined pauseReading?: () => void resumeReading?: () => void send: (message: Record, beforeDispatch?: () => Promise) => Promise @@ -99,7 +102,8 @@ export async function openClaudeStreamJsonConnection( queryImpl?: typeof ClaudeAgentSdk.query ): Promise { const { query } = await loadClaudeAgentSdk() - const spawner = createClaudeCodeProcessSpawn(spawnImpl, process.platform, launch) + const startup = claudeStartupAnswerObservation() + const spawner = createClaudeCodeProcessSpawn(spawnImpl, process.platform, launch, startup.attach) // Always supply scrubbed env to prevent ambient auth leaks; only launch.env may pin CLAUDE_CONFIG_DIR. const env = await withoutRejectedClaudeApiKey( withoutKeys( @@ -128,6 +132,7 @@ export async function openClaudeStreamJsonConnection( if (!child || !managed) { throw new Error('the claude agent SDK returned without spawning a child') } + const exited = Promise.withResolvers() let closing = false let terminalError: Error | null = null let faultReported = false @@ -186,6 +191,7 @@ export async function openClaudeStreamJsonConnection( if (managed.rootExitObserved && !exitReported) { exitReported = true handlers.onExit?.(terminalError, { expected: closing }) + exited.resolve(terminalError) } } @@ -252,6 +258,7 @@ export async function openClaudeStreamJsonConnection( ) ) } + startup.finish() // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: dispatch constructs the SDK user envelope after mapping every content block. return inbox.push(message as unknown as SDKUserMessage, beforeDispatch) } @@ -285,13 +292,17 @@ export async function openClaudeStreamJsonConnection( pid: spawner.pid }) } + startup.finish() return true })() return closePromise } + const controls = createClaudeControlSurface(session) return { - ...createClaudeControlSurface(session), + ...controls, + startupFailure: startup.failure, + exited: exited.promise, get pid() { return spawner.pid }, diff --git a/src/main/claude/fixtures/startup-sdk-process-exit.json b/src/main/claude/fixtures/startup-sdk-process-exit.json new file mode 100644 index 000000000000..b8329e128e11 --- /dev/null +++ b/src/main/claude/fixtures/startup-sdk-process-exit.json @@ -0,0 +1,22 @@ +{ + "sdk": "0.3.284", + "probe": "actual CLI invalid startup flag before initialize", + "records": [ + { + "initializationError": { + "name": "Error", + "message": "Claude Code process exited with code 1", + "errorClass": "process_exited_nonzero", + "exitCode": 1 + } + }, + { + "iteratorError": { + "name": "Error", + "message": "Claude Code process exited with code 1", + "errorClass": "process_exited_nonzero", + "exitCode": 1 + } + } + ] +} diff --git a/src/main/claude/structured/claude-structured-session-acquisition.ts b/src/main/claude/structured/claude-structured-session-acquisition.ts index 6a79864e23b8..691142e6a15c 100644 --- a/src/main/claude/structured/claude-structured-session-acquisition.ts +++ b/src/main/claude/structured/claude-structured-session-acquisition.ts @@ -24,7 +24,10 @@ import { } from './claude-structured-session-publication' import type * as sessionState from './claude-structured-session-state' import { resolveClaudeAcquisitionError } from './claude-structured-session-close' -import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { + startupAcquisitionRefusal, + withObservedProviderExit +} from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import { readClaudeTranscriptEntryUuid } from '../claude-transcript-entry-uuid' import { claudeTurnResumePointWriter } from './claude-structured-resume-point' import { withAgentSessionCreatePhase } from '../../observability/agent-session-instrumentation' @@ -293,7 +296,7 @@ export async function acquireClaudeSession({ } catch (error) { unbindReadingControl?.() throw await resolveClaudeAcquisitionError({ - error, + error: startupAcquisitionRefusal(error) ?? attempt.connection?.startupFailure?.() ?? error, sessionId, sessions, attempt, diff --git a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts index 5a273542b870..b2308de103e9 100644 --- a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts @@ -1,3 +1,4 @@ +import { withTimeout } from '../../../shared/promise-timeout-fallback' import { agentSessionFailureFact, providerDiagnosticOf @@ -46,7 +47,11 @@ export function observeClaudeSessionExit( session.events?.observeExit?.(claudeHeldSends(session), observedAt) lifecycle.sessions.delete(sessionId) // Run tree cleanup before recovery; an exit callback proves only root exit. - const closePromise = session.connection.close().catch(() => false) + const cleanup = session.connection.close().catch(() => false) + const closePromise = + session.dispatchSequence === 0 && providerExitObserved(error) + ? withTimeout(cleanup, 2_000, false) + : cleanup const exit: ClaudeSessionExit = { connection: session.connection, session, @@ -81,10 +86,12 @@ export function settleClaudeUnexpectedExit( settleClaudeExitedSession(exit.session) return } - // Save the completed-turn cursor before recovery permits reacquisition. - await persistClaudeSessionHandle(sessionId, exit.session, deps).catch((error: unknown) => { - console.warn('[claude-resume-point] exit cursor was not persisted:', { sessionId, error }) - }) + // Without a dispatch, acquisition's saved cursor is still authoritative. + if (exit.session.dispatchSequence !== 0) { + await persistClaudeSessionHandle(sessionId, exit.session, deps).catch((error: unknown) => { + console.warn('[claude-resume-point] exit cursor was not persisted:', { sessionId, error }) + }) + } if (exits.get(sessionId) !== exit) { settleClaudeExitedSession(exit.session) return @@ -92,6 +99,8 @@ export function settleClaudeUnexpectedExit( exits.delete(sessionId) const report = claudeCloseReport(exit.connection, true, exit.error) lifecycle.settledExitErrors.set(sessionId, report ?? exit.error) + const startupFailure = + exit.session.dispatchSequence === 0 ? exit.connection.startupFailure?.() : undefined const ended: ClaudeStructuredSessionEvent = { type: 'ended', sessionId, @@ -100,11 +109,13 @@ export function settleClaudeUnexpectedExit( failure: startupAcquisitionRefusal(exit.error) || providerSpawnAbsent(exit.error) ? providerStartupFailureFact(exit.error) - : providerExitObserved(exit.error) - ? agentSessionFailureFact('providerExited', { - detail: providerDiagnosticOf(exit.error) - }) - : agentSessionFailureFact('hostFault'), + : startupFailure + ? providerStartupFailureFact(startupFailure) + : providerExitObserved(exit.error) + ? agentSessionFailureFact('providerExited', { + detail: providerDiagnosticOf(exit.error) + }) + : agentSessionFailureFact('hostFault'), cause: 'unexpected-exit', ...(report instanceof AgentSessionAcquisitionRootExitObservedError && report.deathEvidenceKind === 'kill-delivered' diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts new file mode 100644 index 000000000000..22deb16a4108 --- /dev/null +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -0,0 +1,139 @@ +import sdkExit from '../fixtures/startup-sdk-process-exit.json' +import { afterEach, expect, it, vi } from 'vitest' +import { createFakePipedChild } from '../../../shared/__fixtures__/fake-spawned-child' +import { claudeStartupAnswerObservation } from '../claude-startup-answer-observation' +import { runClaudeControl } from '../claude-agent-sdk-control-requests' +import { retryableStructuredAgentSessionStart } from '../../native-chat/agent-session-wire/structured-agent-session-start-retry' +import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { + adapterAtPublishFor, + fakeClaude, + identityFor +} from './claude-structured-session-test-support' +import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' + +afterEach(() => vi.useRealTimers()) + +it.each(['control_request_failed', 'process_exited_nonzero'])( + 'preserves the independently pending SDK %s outcome when exit proof wins first', + async (errorClass) => { + vi.useFakeTimers() + const claude = fakeClaude({ holdInit: true }) + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterAtPublishFor(claude, {}, events) + const acquired = await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'race' + }) + const connection = claude.connections[0] + const pending = Promise.withResolvers() + connection.initializationResult = () => + runClaudeControl('initialize', () => pending.promise, null) + // Delay SDK cleanup, independently of the actual process exit. + connection.close = () => new Promise(() => {}) + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + const exit = withObservedProviderExit(new Error('claude exited')) + connection.handlers.onExit?.(exit, { expected: false }) + await vi.advanceTimersByTimeAsync(10) + const recorded = sdkExit.records[0].initializationError + if (!recorded) { + throw new Error('missing recorded SDK exit') + } + pending.reject( + errorClass === 'process_exited_nonzero' + ? Object.assign(new Error(recorded.message), recorded) + : Object.assign(new Error('SDK terminal outcome'), { errorClass, exitCode: 1 }) + ) + await vi.advanceTimersByTimeAsync(2_000) + const error = await result + expect(error.cause).toMatchObject({ errorClass, exitCode: 1 }) + expect(retryableStructuredAgentSessionStart(error)).toBe( + errorClass === 'process_exited_nonzero' + ) + expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) + claude.dispose() + await adapter.closeAll() + } +) + +it('retains a failed result before a stalled SDK transcript flush and never waits on startup cursor persistence', async () => { + vi.useFakeTimers() + const claude = fakeClaude({ holdInit: true }) + const persist = vi.fn(() => new Promise(() => {})) + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterAtPublishFor(claude, {}, events, [], undefined, persist) + const acquired = await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'flush' }) + const connection = claude.connections[0] + connection.close = () => new Promise(() => {}) + const raw = createFakePipedChild() + const observation = claudeStartupAnswerObservation() + observation.attach(raw) + connection.startupFailure = observation.failure + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + raw.stdout.write( + `${JSON.stringify({ + type: 'result', + is_error: true, + subtype: 'error_during_execution', + errors: ['startup refused'] + })}\n` + ) + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(withObservedProviderExit(new Error('exited')), { expected: false }) + await vi.advanceTimersByTimeAsync(2_100) + const error = await result + expect(error.errorClass).toBe('error_result') + expect(retryableStructuredAgentSessionStart(error)).toBe(false) + expect(persist).not.toHaveBeenCalled() + expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) + claude.dispose() + await adapter.closeAll() +}) + +it('settles an unanswered Claude startup exit with the same bounded recovery eligibility', async () => { + vi.useFakeTimers() + const claude = fakeClaude({ holdInit: true }) + const adapter = adapterAtPublishFor(claude) + const acquired = await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'no-answer' + }) + const connection = claude.connections[0] + connection.close = () => new Promise(() => {}) + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + const error = withObservedProviderExit(new Error('child exited before initialize')) + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(error, { expected: false }) + await vi.advanceTimersByTimeAsync(2_100) + expect(await result).toBe(error) + expect(retryableStructuredAgentSessionStart(await result)).toBe(true) + claude.dispose() + await adapter.closeAll() +}) diff --git a/src/main/claude/structured/claude-structured-session-startup.ts b/src/main/claude/structured/claude-structured-session-startup.ts index 29d41debe1c8..0b67558fd3f9 100644 --- a/src/main/claude/structured/claude-structured-session-startup.ts +++ b/src/main/claude/structured/claude-structured-session-startup.ts @@ -1,3 +1,9 @@ +import { isClaudeQueryClosed } from '../claude-agent-sdk-control-requests' +import { withTimeout } from '../../../shared/promise-timeout-fallback' +import { + startupAcquisitionRefusal, + providerExitObserved +} from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' // Initialize gates host open after process publication; settings enrichment runs independently. import type { AgentSessionAccountKind } from '../../../shared/agent-session/agent-session-availability' @@ -64,22 +70,49 @@ export async function readClaudeStartupFacts(input: { emit: (event: ClaudeStructuredSessionEvent) => void }): Promise { let timer: ReturnType | undefined + const control = input.connection.initializationResult().then((result) => { + const authError = claudeInitializationAuthError(result, input.account) + if (authError) { + throw authError + } + return result + }) const initialization = await Promise.race([ - input.connection - .initializationResult() - .catch((error: unknown) => { - // The SDK may replace the child's own exit diagnostic with a generic closed-query error. + control.catch((error: unknown) => { + try { input.initProof.seen() - throw error - }) - .then((result) => { - const authError = claudeInitializationAuthError(result, input.account) - if (authError) { - throw authError + } catch (proofError) { + if (startupAcquisitionRefusal(proofError)) { + throw proofError + } + if (isClaudeQueryClosed(error)) { + throw input.connection.startupFailure?.() ?? proofError } - return result - }), - input.initProof.promise.then(() => new Promise(() => {})), + } + throw input.connection.startupFailure?.() ?? error + }), + Promise.race([ + input.initProof.promise.then(() => new Promise(() => {})), + input.connection.exited?.then((error) => { + throw error + }) ?? new Promise(() => {}) + ]).catch(async (error: unknown) => { + if (startupAcquisitionRefusal(error) || !providerExitObserved(error)) { + throw error + } + const outcome = await withTimeout( + control.then( + () => ({ error }), + (error: unknown) => ({ error }) + ), + 2_000, + { error } + ) + throw ( + input.connection.startupFailure?.() ?? + (isClaudeQueryClosed(outcome.error) ? error : outcome.error) + ) + }), new Promise((_resolve, reject) => { timer = setTimeout( () => @@ -97,7 +130,10 @@ export async function readClaudeStartupFacts(input: { }) ]).finally(() => clearTimeout(timer)) if (input.connection.closed) { - throw new Error('claude session closed before startup completed') + throw ( + input.connection.startupFailure?.() ?? + new Error('claude session closed before startup completed') + ) } input.emit({ type: 'options', @@ -215,7 +251,10 @@ export async function openClaudeSession(input: { const { session } = input const initialized = await input.facts if (!input.isCurrent()) { - throw new Error('claude session closed before startup completed') + throw ( + session.connection.startupFailure?.() ?? + new Error('claude session closed before startup completed') + ) } initialized.initProof.onRefusal = (error) => { if (input.isCurrent()) { diff --git a/src/main/codex/app-server/codex-app-server-record-dispatch.ts b/src/main/codex/app-server/codex-app-server-record-dispatch.ts index 8f595d3d2317..ef50c49d2682 100644 --- a/src/main/codex/app-server/codex-app-server-record-dispatch.ts +++ b/src/main/codex/app-server/codex-app-server-record-dispatch.ts @@ -118,7 +118,8 @@ export function createCodexAppServerRecordDispatcher(input: { waiter.method, typeof error.code === 'number' ? error.code : null, `codex app-server ${waiter.method} failed: ${detail}`, - typeof error.message === 'string' ? error.message : undefined + typeof error.message === 'string' ? error.message : undefined, + error.data ) ) return diff --git a/src/main/codex/app-server/codex-app-server-request-error.ts b/src/main/codex/app-server/codex-app-server-request-error.ts index fca7c7e1081e..42ef064901db 100644 --- a/src/main/codex/app-server/codex-app-server-request-error.ts +++ b/src/main/codex/app-server/codex-app-server-request-error.ts @@ -7,15 +7,18 @@ import { export class CodexAppServerRequestError extends Error { /** Codex's own `error.message`, kept apart from the Orca text around it. */ readonly providerDiagnostic?: ProviderDiagnostic + readonly data?: unknown constructor( readonly method: string, readonly code: number | null, message: string, - providerMessage?: string + providerMessage?: string, + data?: unknown ) { super(message) this.name = 'CodexAppServerRequestError' + Object.defineProperty(this, 'data', { value: data, enumerable: false }) const diagnostic = providerMessage === undefined ? undefined : providerDiagnostic(providerMessage, 'person') if (diagnostic) { diff --git a/src/main/codex/app-server/codex-startup-failure.ts b/src/main/codex/app-server/codex-startup-failure.ts new file mode 100644 index 000000000000..ecdf33548413 --- /dev/null +++ b/src/main/codex/app-server/codex-startup-failure.ts @@ -0,0 +1,46 @@ +import type { CodexAppServerRequestError } from './codex-app-server-request-error' + +/** Only declarations whose provider contract permits replay of a start. */ +export function temporaryCodexStartupAnswer(error: CodexAppServerRequestError): boolean { + const message = error.providerDiagnostic?.text ?? error.message + if (error.code === -32001 && message === 'Server overloaded; retry later.') { + return true + } + if (error.method !== 'thread/start' && error.method !== 'thread/resume') { + return false + } + const data = typeof error.data === 'object' && error.data !== null ? error.data : {} + if (error.code === -32600) { + if ('reason' in data && data.reason === 'serverShuttingDown') { + return true + } + if (message === 'Workspace change in progress; retry when it completes') { + return true + } + if ( + error.method === 'thread/resume' && + /^thread .+ is closing; retry thread\/resume after the thread is closed$/.test(message) + ) { + return true + } + if ( + error.method === 'thread/start' && + 'reason' in data && + data.reason === 'cloudConfigBundle' && + !('action' in data && data.action === 'relogin') && + 'errorCode' in data + ) { + return ( + data.errorCode === 'Timeout' || + (data.errorCode === 'RequestFailed' && + 'detail' in data && + data.detail === 'Failed to load cloud config bundle (workspace-managed policies).') + ) + } + } + return ( + error.method === 'thread/start' && + error.code === -32603 && + /^error creating thread: .*thread-store retryable error: /s.test(message) + ) +} diff --git a/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json b/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json new file mode 100644 index 000000000000..f04ec730b360 --- /dev/null +++ b/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json @@ -0,0 +1,6 @@ +{ + "capture": "Native resume failure logged on 2026-10-30; only the account path is sanitized. The original dispatcher did not retain error.data, so it is unknown here.", + "method": "thread/resume", + "code": -32600, + "message": "paginated rollout path `/qa-home/.codex/sessions/rollout-2026-10-28T18-52-57.454Z-01a183db-b8d6-7170-af39-e004651c1e1e.jsonl` does not have a canonical rollout filename" +} diff --git a/src/main/codex/codex-scripted-child.test-fixture.ts b/src/main/codex/codex-scripted-child.test-fixture.ts index 5d40a7a41751..3b64304a5d32 100644 --- a/src/main/codex/codex-scripted-child.test-fixture.ts +++ b/src/main/codex/codex-scripted-child.test-fixture.ts @@ -1,3 +1,7 @@ +import { + providerDiagnostic, + withProviderDiagnostic +} from '../../shared/agent-session/failures/agent-session-failure' import type { ScriptedAgentChild } from '../runtime/structured-agent-scripted-child.test-fixture' import type { CodexAppServerConnection } from './app-server/codex-app-server-connection-types' import type { openCodexAppServerConnection } from './app-server/codex-app-server-connection' @@ -55,6 +59,7 @@ export const codexScriptedChild = ( let closed = 0 const prompts: string[] = [] const requests: { method: string; params?: Record }[] = [] + let exitNewest: ((message: string) => void) | undefined let newest: ReturnType> | undefined const open: typeof openCodexAppServerConnection = async (_launch, handlers = {}) => { notify = (method, params) => handlers.onNotification?.(method, params) @@ -153,6 +158,12 @@ export const codexScriptedChild = ( return true } } + exitNewest = (message) => { + ended = true + const error = withProviderDiagnostic(new Error(message), providerDiagnostic(message, 'log')) + gate.reject(error) + handlers.onExit?.(error, { expected: false }) + } // A gate can be closed before initialization starts or after it has already answered. void gate.promise.catch(() => {}) await handlers.onSpawned?.(connection.pid ?? 0) @@ -185,6 +196,7 @@ export const codexScriptedChild = ( completeTurns = value }, releaseHandshake: () => newest?.resolve({}), + exitHandshake: (message) => exitNewest?.(message), failHandshake: (message) => newest?.reject(new CodexAppServerRequestError('initialize', -32603, message, message)), prompts: () => prompts, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts index 021844e711fe..e2b344d2b619 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts @@ -1,4 +1,5 @@ -import { sameProviderChild } from './structured-agent-session-provider-child' +import { providerStartupFailureFact } from './structured-agent-session-failure-text' +import { retryableStructuredAgentSessionStart } from './structured-agent-session-start-retry' import { waitForPromiseWithSignal } from '../../../shared/abort-signal-reason' import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' import { @@ -22,10 +23,13 @@ export async function openStructuredAgentSessionProviderChild( if (!context.runtimeState.startupAttempts.onClock(sessionId, child)) { return } - const attempt = context.runtimeState.acquireAborts.begin(sessionId, false, false, () => { - const current = context.sessions.get(sessionId)?.child - return current !== undefined && current !== null && sameProviderChild(current, child) - }) + // Physical exit ends readiness; it must not erase the outstanding protocol answer. + const attempt = context.runtimeState.acquireAborts.begin(sessionId) + const observation = context.runtimeState.startupAttempts.observeOpen( + sessionId, + child, + attempt.abort + ) const openedAt = context.now() try { attempt.signal.throwIfAborted() @@ -51,6 +55,13 @@ export async function openStructuredAgentSessionProviderChild( attempt.signal ) } catch (error) { + if (!attempt.signal.aborted) { + observation.failed( + providerStartupFailureFact(error), + retryableStructuredAgentSessionStart(error), + error + ) + } // Aborted startup already has a Stop/close/quit/expiry owner. if (!attempt.signal.aborted) { const openMs = Math.max(0, context.now() - openedAt) @@ -65,6 +76,7 @@ export async function openStructuredAgentSessionProviderChild( ) } } finally { + observation.finish() attempt.end() } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts index a75997ad6f45..4fb0f03c6917 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts @@ -1,3 +1,5 @@ +import { retryableStructuredAgentSessionStart } from './structured-agent-session-start-retry' +import { withTimeout } from '../../../shared/promise-timeout-fallback' import { continuingStructuredAgentSessionInput } from './structured-agent-session-handing-over' import { randomUUID } from 'node:crypto' import { setTimeout as delay } from 'node:timers/promises' @@ -5,7 +7,10 @@ import { throwIfSignalAborted, waitForPromiseWithSignal } from '../../../shared/ import { refuseUnclassified } from '../../../shared/agent-session/failures/agent-session-wire-refusals' import { agentSessionWriteNoticeEnglish } from '../../../shared/agent-session/failures/agent-session-refusal-notice' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session/requests/agent-session-queued-submission' -import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { + agentSessionFailureFact, + type SubmissionRejectionFact +} from '../../../shared/agent-session/failures/agent-session-failure' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionAttachLock } from './structured-agent-session-attach-lock' import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-agent-start' @@ -18,6 +23,8 @@ import { recordStructuredAgentSessionStartFailure } from './structured-agent-session-start-failure-row' import { + providerStartupFailureFact, + withObservedProviderExit, structuredAgentSessionStartFailure, structuredAgentSessionStartFailureFact } from './structured-agent-session-failure-text' @@ -77,6 +84,20 @@ export function registerStructuredAgentSessionStart( if (ready === 'ready') { return result } + if ( + handle.result.opening && + handle.result.retry !== false && + !finalStartupExitFailure(context.sessions.get(sessionId)?.lastEndedChild?.failure) + ) { + await lock.wait( + waitForPromiseWithSignal( + withTimeout(handle.result.opening, 2_000, undefined), + acquire.signal + ) + ) + handle.result.cancelOpen?.() + throwIfSignalAborted(acquire.signal) + } result = await lock.commit(async () => { const session = context.sessions.get(sessionId) const ended = session?.lastEndedChild @@ -94,16 +115,35 @@ export function registerStructuredAgentSessionStart( 'The previous agent process could not be verified as stopped.' ) } + const finalExit = finalStartupExitFailure(ended.failure) + const openError = handle.result.openError + const openFailure = + ended.cause === 'exit' && openError instanceof Error + ? providerStartupFailureFact(withObservedProviderExit(openError)) + : handle.result.failure const failure = structuredAgentSessionStartFailureFact({ - exit: handle.result.failure ?? ended.failure + exit: finalExit + ? ended.failure + : openFailure + ? { ...openFailure, detail: openFailure.detail ?? ended.failure?.detail } + : ended.failure }) return { ...failedStructuredAgentSessionStart(), startKey: ended.generation, failure, retry: - handle.result.retry ?? - (ended.cause === 'exit' && failure.kind === 'providerStartFailed') + !finalExit && + ('openError' in handle.result + ? retryableStructuredAgentSessionStart( + handle.result.openError, + ended.cause === 'exit' + ) + : (handle.result.retry ?? + (ended.cause === 'exit' && + failure.kind === 'providerStartFailed' && + !failure.refusal && + !failure.requestTimeout))) } }) } @@ -208,3 +248,12 @@ export function structuredAgentSessionStartStillQueued( ) === true ) } + +function finalStartupExitFailure(failure: SubmissionRejectionFact | undefined): boolean { + return ( + !!failure && + (!['providerExited', 'providerStartFailed', 'startFailed'].includes(failure.kind) || + !!failure.refusal || + !!failure.requestTimeout) + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts index 853a0d61e175..d990d91b9276 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts @@ -1,3 +1,6 @@ +import canonicalFilenameRefusal from '../../codex/app-server/fixtures/start-resume-canonical-filename-refusal.json' +import sdkExit from '../../claude/fixtures/startup-sdk-process-exit.json' +import { CodexAppServerUnsupportedError } from '../../codex/app-server/codex-app-server-session' import { PassThrough } from 'node:stream' import { describe, expect, it, vi } from 'vitest' import { AcpAgentError, AcpInvalidResponseError, AcpRpcError } from '../../acp/acp-errors' @@ -40,16 +43,19 @@ function expectExitEvidenceRequired(error: unknown) { ).toBe(false) } -function expectFinalWithExitEvidence(error: Error) { +function expectFinalWithExitEvidence(error: unknown) { expect(retryableStructuredAgentSessionStart(error)).toBe(false) expect( retryableStructuredAgentSessionAcquisition(new AgentSessionAcquisitionExitProvenError(error)) ).toBe(false) + if (!(error instanceof Error)) { + throw new Error('expected Error') + } expect(retryableStructuredAgentSessionStart(withObservedProviderExit(error))).toBe(false) } describe('recorded command rejection wrappers at the startup classifier', () => { - it('keeps recorded Claude permission rejection retryable without a typed startup refusal', async () => { + it('keeps a recorded Claude control rejection final, including after exit', async () => { const capture = captures.claude // SDK rejection is supplied here; this does not exercise the SDK's wire parser. const error = await runClaudeControl(capture.request.request.subtype, () => @@ -60,10 +66,10 @@ describe('recorded command rejection wrappers at the startup classifier', () => subtype: capture.request.request.subtype, message: capture.response.response.error }) - expectExitEvidenceRequired(error) + expectFinalWithExitEvidence(error) }) - it('keeps the recorded Codex invalid thread/read answer retryable outside resume', async () => { + it('keeps a recorded Codex answer final outside resume too', async () => { const capture = captures.codex const pending = Promise.withResolvers() const failed = pending.promise.catch((caught: unknown) => caught) @@ -85,10 +91,10 @@ describe('recorded command rejection wrappers at the startup classifier', () => code: capture.response.error.code, providerDiagnostic: { text: capture.response.error.message, audience: 'person' } }) - expectExitEvidenceRequired(error) + expectFinalWithExitEvidence(error) }) - it('keeps the recorded Pi model rejection retryable without a typed startup refusal', async () => { + it('keeps a recorded Pi rejection final, including after exit', async () => { const capture = captures.pi const input = new PassThrough() const output = new PassThrough() @@ -108,7 +114,7 @@ describe('recorded command rejection wrappers at the startup classifier', () => command: capture.response.command, message: capture.response.error }) - expectExitEvidenceRequired(error) + expectFinalWithExitEvidence(error) } finally { peer.close() input.destroy() @@ -116,7 +122,7 @@ describe('recorded command rejection wrappers at the startup classifier', () => } }) - it('keeps the recorded ACP -32603 retryable regardless of its InvalidRequestError data', async () => { + it('keeps a recorded ACP permanent rejection final', async () => { const capture = captures.acp const pending = Promise.withResolvers() const failed = pending.promise.catch((caught: unknown) => caught) @@ -126,7 +132,7 @@ describe('recorded command rejection wrappers at the startup classifier', () => expect(error).toBeInstanceOf(AcpAgentError) expect(error).toMatchObject(capture.response.error) expect(diagnose).not.toHaveBeenCalled() - expectExitEvidenceRequired(error) + expectFinalWithExitEvidence(error) }) it.each([ @@ -167,8 +173,8 @@ describe('recorded command rejection wrappers at the startup classifier', () => expectFinalWithExitEvidence(error) }) - it.each([-32600, -32603])('keeps unknown Codex resume answers retryable: %s', (code) => { - expectExitEvidenceRequired( + it.each([-32600, -32603])('keeps unknown Codex resume answers final: %s', (code) => { + expectFinalWithExitEvidence( new CodexAppServerRequestError('thread/resume', code, 'Server is temporarily unavailable') ) }) @@ -193,3 +199,129 @@ describe('recorded command rejection wrappers at the startup classifier', () => ) }) }) + +// Source-shaped declarations: these provider-side temporary conditions are not locally reproducible. +it('keeps the recorded canonical rollout filename refusal final on the first answer', () => { + const { method, code, message } = canonicalFilenameRefusal + expectFinalWithExitEvidence(new CodexAppServerRequestError(method, code, message, message)) +}) + +it('preserves an actual SDK nonzero exit through initialize control cleanup', async () => { + const recorded = sdkExit.records[0].initializationError + if (!recorded) { + throw new Error('missing recorded initialization error') + } + const cause = Object.assign(new Error(recorded.message), recorded) + const error = await runClaudeControl('initialize', () => Promise.reject(cause)).catch( + (error) => error + ) + expect(error.cause).toBe(cause) + expectExitEvidenceRequired(error) +}) + +it('keeps unsupported Codex methods final after process exit too', () => { + expectFinalWithExitEvidence(new CodexAppServerUnsupportedError('method unsupported')) +}) + +describe('positive provider startup retry contracts', () => { + const codex = (method: string, code: number, message: string, data?: unknown) => + new CodexAppServerRequestError(method, code, message, message, data) + it.each([ + codex('initialize', -32001, 'Server overloaded; retry later.'), + codex('thread/start', -32600, 'Server is draining; retry after reconnecting', { + reason: 'serverShuttingDown' + }), + codex('thread/resume', -32600, 'Server is draining; retry after reconnecting', { + reason: 'serverShuttingDown' + }), + codex('thread/start', -32600, 'Workspace change in progress; retry when it completes'), + codex( + 'thread/resume', + -32600, + 'thread abc is closing; retry thread/resume after the thread is closed' + ), + codex('thread/start', -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode: 'Timeout' + }), + ...[undefined, 503, 403].map((statusCode) => + codex('thread/start', -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode: 'RequestFailed', + statusCode, + detail: 'Failed to load cloud config bundle (workspace-managed policies).' + }) + ), + codex( + 'thread/start', + -32603, + 'error creating thread: storage failed: thread-store retryable error: busy' + ), + new AcpAgentError(-32603, 'server unavailable', { errorName: 'ServerUnavailable' }), + new AcpAgentError(-32603, 'catalog not ready', { errorName: 'CatalogNotReady' }) + ])('retries only after separately proving absence: $message', expectExitEvidenceRequired) + + it.each([ + ...['Auth', 'InvalidBundle', 'Internal', 'RequestFailed'].map((errorCode) => + codex('thread/start', -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode + }) + ), + codex('thread/start', -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode: 'Timeout', + action: 'relogin' + }), + codex('thread/start', -32603, 'error creating thread: outcome unknown'), + codex('thread/start', -32603, 'error creating thread: internal error'), + codex('thread/resume', -32600, 'please retry this unknown answer') + ])( + 'does not infer retry from an error code or a generic message: $message', + expectFinalWithExitEvidence + ) + + it.each([ + ['control_request_failed', false], + ['process_exited_nonzero', true], + ['process_killed_by_signal', true], + ['error_result', false], + ['initialize_timeout', false], + ['executable_not_found', false], + ['executable_launch_failed', false], + ['aborted', false], + ['unreadable', false], + ['future_unknown_tag', false], + [undefined, false] + ])('preserves SDK %s through the control wrapper', async (errorClass, retry) => { + const cause = Object.assign(new Error('SDK error'), errorClass ? { errorClass } : {}) + const error = await runClaudeControl('initialize', () => Promise.reject(cause)).catch( + (error) => error + ) + expect(error.cause).toBe(cause) + expect(retryableStructuredAgentSessionStart(error)).toBe(retry) + if (!retry) { + expectFinalWithExitEvidence(error) + } + }) + + it('preserves Codex error.data at the response boundary', async () => { + const pending = Promise.withResolvers() + const result = pending.promise.catch((error) => error) + const dispatcher = createCodexAppServerRecordDispatcher({ + handlers: {}, + writeResponse: vi.fn(), + onProtocolFailure: vi.fn() + }) + dispatcher.addPending(1, { method: 'thread/start', ...pending, timer: undefined }) + const data = { reason: 'serverShuttingDown' } + dispatcher.dispatch({ + id: 1, + error: { code: -32600, message: 'Server is draining; retry after reconnecting', data } + }) + const error = await result + expect(error.data).toBe(data) + expect(Object.keys(error)).not.toContain('data') + expectExitEvidenceRequired(error) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts index 93dec21f6507..63b035e1175a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts @@ -1,6 +1,9 @@ import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' -import { ClaudeControlRequestError } from '../../claude/claude-agent-sdk-control-requests' -import { AcpAgentError, AcpRpcError } from '../../acp/acp-errors' +import { claudeStartupAnswer } from '../../claude/claude-startup-failure' +import { temporaryCodexStartupAnswer } from '../../codex/app-server/codex-startup-failure' +import { CodexAppServerUnsupportedError } from '../../codex/app-server/codex-app-server-session' +import { temporaryAcpStartupAnswer } from '../../acp/acp-startup-failure' +import { AcpRpcError } from '../../acp/acp-errors' import { JsonlRpcResponseError } from '../../jsonl-rpc/peer' import { findInErrorChain } from '../../provider-process/provider-error-chain' import { @@ -19,33 +22,24 @@ import { } from './structured-agent-session-adapter' import { providerExitObserved } from './structured-agent-session-failure-text' -function finalAgentStartAnswer(error: Error): boolean { - if (error instanceof AcpRpcError && !(error instanceof AcpAgentError)) { - return true +function agentStartAnswer(error: Error): 'temporary' | 'final' | undefined { + if (error instanceof CodexAppServerRequestError) { + return temporaryCodexStartupAnswer(error) ? 'temporary' : 'final' } - if (!(error instanceof AcpAgentError || error instanceof CodexAppServerRequestError)) { - return false - } - if (error.code === -32601 || error.code === -32602) { - return true + if (error instanceof CodexAppServerUnsupportedError || error instanceof JsonlRpcResponseError) { + return 'final' } - if ( - !(error instanceof CodexAppServerRequestError) || - error.method !== 'thread/resume' || - error.code !== -32600 - ) { - return false + if (error instanceof AcpRpcError) { + return temporaryAcpStartupAnswer(error) ? 'temporary' : 'final' } - const message = - error.providerDiagnostic?.text ?? - error.message.replace(/^codex app-server thread\/resume failed: /, '') - return /^(?:invalid (?:thread|session) id:|thread not found(?::|$)|no rollout found for thread id )/.test( - message - ) + return claudeStartupAnswer(error) } /** Only positive provider evidence retries; an Orca fault or a typed refusal stays final. */ -export function retryableStructuredAgentSessionStart(error: unknown): boolean { +export function retryableStructuredAgentSessionStart( + error: unknown, + unexpectedExit = false +): boolean { if ( providerRequestTimeoutOf(error) || argumentProblemOf(error) || @@ -60,7 +54,7 @@ export function retryableStructuredAgentSessionStart(error: unknown): boolean { cause instanceof AgentSessionAcquisitionExitUnprovenError || cause instanceof AgentSessionAcquisitionRefusal || isAgentSessionPreSpawnError(cause) || - finalAgentStartAnswer(cause) + agentStartAnswer(cause) === 'final' ? true : undefined ) @@ -68,12 +62,10 @@ export function retryableStructuredAgentSessionStart(error: unknown): boolean { return false } return ( + unexpectedExit || providerExitObserved(error) || findInErrorChain(error, (cause) => - cause instanceof CodexAppServerRequestError || - cause instanceof ClaudeControlRequestError || - cause instanceof AcpAgentError || - cause instanceof JsonlRpcResponseError || + agentStartAnswer(cause) === 'temporary' || ('code' in cause && ['EAGAIN', 'EMFILE', 'ENFILE'].some((code) => code === cause.code)) ? true : undefined diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts index eafb3d3084b0..9de29b0b463a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts @@ -38,6 +38,7 @@ vi.mock('node:timers/promises', async (importOriginal) => { let host: ReturnType['host'] let acquire: ReturnType['acquire'] let dispatch: ReturnType['dispatch'] +const openFailures = new Map void>() const openGates = new Map void>() const eventually = (assertion: () => void | Promise) => vi.waitFor(assertion, { timeout: 10_000, interval: 0 }) @@ -52,8 +53,10 @@ beforeEach(async () => { await host.close(SESSION, 'evict') acquire.mockClear() openGates.clear() + openFailures.clear() host.deps.adapter.open = ({ acquisitionGeneration }) => - new Promise((resolve) => { + new Promise((resolve, reject) => { + openFailures.set(acquisitionGeneration!, reject) openGates.set(acquisitionGeneration!, () => resolve({ reportedOptions: {} })) }) }) @@ -116,6 +119,8 @@ async function endAttempt() { reason: 'exited while starting', cause: 'unexpected-exit' }) + openFailures.get(child.generation)?.(new Error('provider exited before open answered')) + openFailures.delete(child.generation) await eventually(() => expect(session.child).toBeNull()) } @@ -369,6 +374,7 @@ describe('final logical startup settlement', () => { expect(runtimeState.currentEventSink(SESSION)).toBeUndefined() acquire.mockClear() openGates.clear() + openFailures.clear() const other = 'unrelated-chat' const unrelated = hostTestAttachParams(null, { providerHandle: undefined, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts index eddaae0b0256..93ed23dd44d8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts @@ -46,8 +46,10 @@ export let acquire: Mock export let dispatch: Mock export let closeSession: Mock> let setOption: Mock +export const providerOpen = vi.fn>() export const providerStarted: Mock = vi.fn() export const recordLiveListing: Mock = vi.fn() +const openFailures = new Map void>() const openGates = new Map void>() export async function proveOpen(model = 'default'): Promise { @@ -90,12 +92,7 @@ async function startHost({ startupLimits, supportsGoals }: StartupHostOptions = logger: createStructuredAgentSessionLogger(), store, adapter: { - open: ({ acquisitionGeneration }) => - new Promise((resolve) => { - openGates.set(acquisitionGeneration!, (model) => - resolve({ reportedOptions: { model, reportedPermissionMode: 'ask' } }) - ) - }), + open: providerOpen, acquire, dispatch, closeSession, @@ -126,6 +123,17 @@ async function startHost({ startupLimits, supportsGoals }: StartupHostOptions = ...(startupLimits === undefined ? {} : { startupLimits }), now: () => NOW }) + const handleEvent = host.handleAdapterEvent.bind(host) + host.handleAdapterEvent = async (event) => { + const result = await handleEvent(event) + if (event.type === 'ended' && event.acquisitionGeneration) { + openFailures.get(event.acquisitionGeneration)?.( + new Error('provider exited before open answered') + ) + openFailures.delete(event.acquisitionGeneration) + } + return result + } } export async function restartWith( @@ -134,6 +142,7 @@ export async function restartWith( ): Promise { await host.close(SESSION, 'evict') await host.flushAllStreamedEvents() + providerOpen.mockClear() acquire.mockImplementation(next) closeSession.mockClear() await startHost(options) @@ -149,7 +158,17 @@ export function registerStartupAttemptHostTests(): void { resetHostTestOperationIds() providerStarted.mockReset() recordLiveListing.mockReset() + openFailures.clear() openGates.clear() + providerOpen.mockReset().mockImplementation( + ({ acquisitionGeneration }) => + new Promise((resolve, reject) => { + openFailures.set(acquisitionGeneration!, reject) + openGates.set(acquisitionGeneration!, (model) => + resolve({ reportedOptions: { model, reportedPermissionMode: 'ask' } }) + ) + }) + ) acquire = vi.fn(async (input) => acquisition(input)) dispatch = vi.fn(async () => ({ state: 'accepted' as const, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts index bd53451c45bc..071586513627 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts @@ -30,6 +30,7 @@ import { host, proveOpen, providerStarted, + providerOpen, publishFirst, restartWith as restartHostWith, stop, @@ -321,6 +322,89 @@ describe('one logical start across physical attempts', () => { return event } + it.each([ + [false, false], + [false, true], + [true, false], + [true, true] + ] as const)( + 'retains a final open answer (exit first: %s, typed refusal: %s)', + async (exitFirst, typed) => { + await restartWith(publishFirst) + const open = Promise.withResolvers() + providerOpen.mockReturnValueOnce(open.promise) + const id = await heldBehindStart('one request') + await eventually(() => expect(providerOpen).toHaveBeenCalledTimes(1)) + const refusal = typed + ? new AgentSessionAcquisitionRefusal('saved conversation missing') + : new CodexAppServerRequestError('thread/resume', -32600, 'canonical filename required') + if (exitFirst) { + await endAttempt() + } + open.reject(refusal) + if (!exitFirst) { + await vi.advanceTimersByTimeAsync(20) + } + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + await vi.advanceTimersByTimeAsync(2_000) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + } + ) + + it.each(['fullAccessAsRoot', 'historyTooLarge'] as const)( + 'keeps the typed %s exit-only refusal final even when raw open never settles', + async (kind) => { + await restartWith(publishFirst) + providerOpen.mockImplementationOnce(() => new Promise(() => {})) + const id = await heldBehindStart('one request') + const child = host.collaboratorsForTests().sessions.get(SESSION)!.child! + await host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + fence: child.fence, + acquisitionGeneration: child.generation!, + reason: 'root refusal', + cause: 'unexpected-exit', + failure: agentSessionFailureFact(kind) + }) + await vi.advanceTimersByTimeAsync(2_100) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect((await submission(id))?.rejection).toMatchObject({ kind }) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + } + ) + + it('bounds an unanswered open after exit and recovers without waiting for the startup ceiling', async () => { + await restartWith(publishFirst) + providerOpen.mockImplementationOnce(() => new Promise(() => {})) + const id = await heldBehindStart('one request') + await endAttempt() + await vi.advanceTimersByTimeAsync(2_200) + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + await proveOpen() + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(dispatch).toHaveBeenCalledTimes(1) + }) + + it('reconnects after a declared-temporary open answer and delivers the held message once', async () => { + await restartWith(publishFirst) + providerOpen.mockRejectedValueOnce( + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) + ) + const id = await heldBehindStart('recover') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + await proveOpen() + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(dispatch).toHaveBeenCalledTimes(1) + }) + function goal() { const change = { kind: 'set' as const, objective: 'finish startup' } return host.changeThreadGoal(CALLER, { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-mint.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-mint.ts new file mode 100644 index 000000000000..253def28bb7c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-mint.ts @@ -0,0 +1,39 @@ +import { randomUUID } from 'node:crypto' +import type { ClaudeAccountRecipe } from '../../../shared/agent-session-acquired-account' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-identity' +import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' +import type { AgentChatPermissionChoice } from '../../../shared/agent-chat-permission-mode' +import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import type { StructuredAgentSessionStartupAttempt } from './structured-agent-session-startup-attempt-contract' + +export function mintStructuredAgentSessionStartupAttempt(input: { + record: AgentSessionRecord + identity: AgentSessionJournalIdentity + spawnToken: string + permissionMode?: AgentChatPermissionChoice + accountRecipe?: ClaudeAccountRecipe + events?: StructuredAgentSessionEventSink + signal?: AbortSignal + startupOptions?: Readonly> + attemptId?: string +}): StructuredAgentSessionStartupAttempt { + const { record } = input + const selected = input.startupOptions ?? record.options + const options = record.provider === 'claude' ? { fastMode: 'false', ...selected } : selected + return { + attemptId: input.attemptId ?? randomUUID(), + identity: input.identity, + fence: record.lease.runtimeFence, + spawnToken: input.spawnToken, + launch: { + location: record.location, + accountHome: record.accountHome, + ...(record.launchDirectory === undefined ? {} : { launchDirectory: record.launchDirectory }) + }, + ...(options ? { options } : {}), + ...(input.permissionMode ? { permissionMode: input.permissionMode } : {}), + ...(input.accountRecipe ? { accountRecipe: input.accountRecipe } : {}), + ...(input.events ? { events: input.events } : {}), + ...(input.signal ? { signal: input.signal } : {}) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts index be9f06e471ae..99d9620c26e1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts @@ -1,5 +1,3 @@ -import type { ClaudeAccountRecipe } from '../../../shared/agent-session-acquired-account' -import { randomUUID } from 'node:crypto' import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' import { waitForStructuredAgentSessionStart, @@ -7,9 +5,6 @@ import { } from './structured-agent-session-start-readiness' export type { StructuredAgentSessionStartReadiness } from './structured-agent-session-start-readiness' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-identity' -import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' -import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' -import type { AgentChatPermissionChoice } from '../../../shared/agent-chat-permission-mode' import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' import { sameProviderChild } from './structured-agent-session-provider-child' import { @@ -18,37 +13,7 @@ import { type StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' -export function mintStructuredAgentSessionStartupAttempt(input: { - record: AgentSessionRecord - identity: AgentSessionJournalIdentity - spawnToken: string - permissionMode?: AgentChatPermissionChoice - accountRecipe?: ClaudeAccountRecipe - events?: StructuredAgentSessionEventSink - signal?: AbortSignal - startupOptions?: Readonly> - attemptId?: string -}): StructuredAgentSessionStartupAttempt { - const { record } = input - const selected = input.startupOptions ?? record.options - const options = record.provider === 'claude' ? { fastMode: 'false', ...selected } : selected - return { - attemptId: input.attemptId ?? randomUUID(), - identity: input.identity, - fence: record.lease.runtimeFence, - spawnToken: input.spawnToken, - launch: { - location: record.location, - accountHome: record.accountHome, - ...(record.launchDirectory === undefined ? {} : { launchDirectory: record.launchDirectory }) - }, - ...(options ? { options } : {}), - ...(input.permissionMode ? { permissionMode: input.permissionMode } : {}), - ...(input.accountRecipe ? { accountRecipe: input.accountRecipe } : {}), - ...(input.events ? { events: input.events } : {}), - ...(input.signal ? { signal: input.signal } : {}) - } -} +export { mintStructuredAgentSessionStartupAttempt } from './structured-agent-session-startup-attempt-mint' export type StructuredAgentSessionExpiredStartup = { sessionId: string @@ -78,7 +43,10 @@ type Tracked = { /** Defer expiry reporting until aborted spawn settles; successful readiness overrides it. */ expiry: StructuredAgentSessionStartupSettled | null failure?: SubmissionRejectionFact + openError?: unknown retry?: boolean + opening?: Promise + cancelOpen?: () => void readiness: PromiseWithResolvers<'ready' | 'ended'> } @@ -124,6 +92,29 @@ export class StructuredAgentSessionStartupAttempts { } } + /** The retained attempt survives child removal, but never records into its replacement. */ + observeOpen( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + cancel: () => void + ) { + const tracked = this.open.get(sessionId) + const completion = Promise.withResolvers() + const matches = tracked?.child && sameProviderChild(tracked.child, child) + if (matches) { + tracked.opening = completion.promise + tracked.cancelOpen = cancel + } + return { + failed: (failure: SubmissionRejectionFact, retry: boolean, openError: unknown): void => { + if (matches) { + Object.assign(tracked, { failure, retry, openError }) + } + }, + finish: (): void => completion.resolve() + } + } + private register(sessionId: string, attemptId: string, registeredAt = this.now()): Tracked { this.end(sessionId) const tracked: Tracked = { diff --git a/src/main/pi/pi-scripted-child.test-fixture.ts b/src/main/pi/pi-scripted-child.test-fixture.ts index 12e72e70b3b3..05f16249195f 100644 --- a/src/main/pi/pi-scripted-child.test-fixture.ts +++ b/src/main/pi/pi-scripted-child.test-fixture.ts @@ -156,6 +156,12 @@ class ScriptedPiConnection implements PiRpcConnection { this.exit(error) } + exitHandshake(message: string): void { + const error = withProviderDiagnostic(new Error(message), providerDiagnostic(message, 'log')) + this.handshake.reject(error) + this.exit(error) + } + async close(): Promise>> { if (!this.closed) { this.onClosed() @@ -288,6 +294,7 @@ export const piScriptedChild = (options: PiScriptOptions = {}): ScriptedPiChild }, releaseHandshake: () => children.at(-1)?.releaseHandshake(), failHandshake: (message) => children.at(-1)?.failHandshake(message), + exitHandshake: (message) => children.at(-1)?.exitHandshake(message), prompts: () => received, spawns: () => children.length, resumes: () => resumed, diff --git a/src/main/runtime/agent-session/structured-agent-session-adapter-start-failure.test.ts b/src/main/runtime/agent-session/structured-agent-session-adapter-start-failure.test.ts index 8ad5905e9908..45f547fe3d78 100644 --- a/src/main/runtime/agent-session/structured-agent-session-adapter-start-failure.test.ts +++ b/src/main/runtime/agent-session/structured-agent-session-adapter-start-failure.test.ts @@ -1,3 +1,4 @@ +import { claudeScriptedChild } from '../../claude/claude-scripted-child.test-fixture' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -32,7 +33,11 @@ afterEach(async () => { } }) -async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgentChild) { +async function failedStart( + agent: 'pi' | 'codex' | 'grok' | 'claude', + script: ScriptedAgentChild, + unanswered = false +) { directory = await mkdtemp(join(tmpdir(), 'orca-adapter-start-failure-')) const root = directory const host = await ensureStructuredAgentSessionHost({ @@ -55,9 +60,11 @@ async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgent agent, providerHandle: undefined, accountHome: - agent === 'pi' - ? { variable: 'PI_CODING_AGENT_DIR', path: root } - : { variable: 'GROK_HOME', path: root }, + agent === 'claude' + ? { variable: 'CLAUDE_CONFIG_DIR', path: root } + : agent === 'pi' + ? { variable: 'PI_CODING_AGENT_DIR', path: root } + : { variable: 'GROK_HOME', path: root }, location: { executionHostId: 'local', wslDistro: null, @@ -88,10 +95,15 @@ async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgent if (!sent.ok) { throw new Error(JSON.stringify(sent.refusal)) } - for (let attempt = 1; attempt <= 5; attempt++) { + const expectedAttempts = unanswered || agent === 'grok' || agent === 'claude' ? 5 : 1 + for (let attempt = 1; attempt <= expectedAttempts; attempt++) { // Retries back off 100·2^n ms (800 ms before the fifth spawn); the 1 s default leaves no room under load. await vi.waitFor(() => expect(script.spawns()).toBe(attempt), { timeout: 5_000 }) - script.failHandshake('provider config is invalid: bad key') + const fail = unanswered ? script.exitHandshake : script.failHandshake + if (!fail) { + throw new Error('fixture has no unanswered exit') + } + fail('provider config is invalid: bad key') } const submission = async () => { await host.flushStreamedEvents(SESSION) @@ -104,7 +116,7 @@ async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgent }) const settled = await submission() const snapshot = await host.journalSnapshot(SESSION) - const failureKind = agent === 'codex' ? 'startFailed' : 'providerStartFailed' + const failureKind = agent === 'codex' && !unanswered ? 'startFailed' : 'providerStartFailed' expect(readWholeAgentSessionFailureFact(settled?.rejection)).toMatchObject({ kind: failureKind, detail: { text: 'provider config is invalid: bad key' } @@ -117,7 +129,7 @@ async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgent ) ).toHaveLength(1) expect(script.prompts()).toEqual([]) - expect(script.spawns()).toBe(5) + expect(script.spawns()).toBe(expectedAttempts) } it('Pi records one failure without dispatch after a failed start', async () => { @@ -129,3 +141,13 @@ it('Codex records one failure without dispatch after a failed start', async () = it('ACP records one failure without dispatch after a failed start', async () => { await failedStart('grok', acpScriptedChild('grok')()) }) + +it('Claude keeps unanswered early exits within five attempts', async () => { + await failedStart('claude', claudeScriptedChild()) +}) +it('Codex keeps unanswered early exits within five attempts', async () => { + await failedStart('codex', codexScriptedChild(), true) +}) +it('Pi keeps unanswered early exits within five attempts', async () => { + await failedStart('pi', piScriptedChild(), true) +}) diff --git a/src/main/runtime/structured-agent-scripted-child.test-fixture.ts b/src/main/runtime/structured-agent-scripted-child.test-fixture.ts index 1e1d8fd3a409..79b551addb11 100644 --- a/src/main/runtime/structured-agent-scripted-child.test-fixture.ts +++ b/src/main/runtime/structured-agent-scripted-child.test-fixture.ts @@ -11,6 +11,8 @@ export type ScriptedAgentChild = { releaseHandshake(): void /** Fails the held handshake of the newest spawn with this provider error; its process exits. */ failHandshake(message: string): void + /** The child dies without answering the pending startup request. */ + exitHandshake?: (message: string) => void /** Default prompts complete immediately; false keeps turns running until interrupt or child exit, modeling background work. */ completeTurns: boolean /** The text of every prompt any spawn received, in order. */ From 66dbdd70005ea9229a7a64ed5ca357f505f36e2e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:21:22 -0800 Subject: [PATCH 02/21] Retire startup opens after exit and keep final answers across exit orders - A start flight cancels its outstanding provider open on every ended path; a direct attach keeps the child-bound abort. - The post-exit answer join is bounded by the remaining startup ceiling. - An elapsed initialize deadline keeps its words when the exit wins the race. - Codex cloud config temporary declarations retry on thread/resume as on thread/start. - A Claude exit before acquisition publishes waits briefly for initialize's answer, so a rejected initialize stays final. - The Claude startup stdout observer parses only failed root results, within the shared line budget, and detaches once it has one. --- .../claude-startup-answer-observation.ts | 35 ++++++-- .../claude-structured-session-acquisition.ts | 8 +- .../claude-structured-session-startup-exit.ts | 38 ++++++++ ...structured-session-startup-failure.test.ts | 87 +++++++++++++++++++ .../claude-structured-session-startup.ts | 15 +--- .../codex/app-server/codex-startup-failure.ts | 1 - .../structured-agent-session-provider-open.ts | 13 ++- ...ructured-agent-session-provider-started.ts | 7 +- ...structured-agent-session-start-attempts.ts | 29 +++++-- ...agent-session-start-retry-recorded.test.ts | 44 +++++----- ...agent-session-startup-attempt-host.test.ts | 45 +++++++++- ...tructured-agent-session-startup-attempt.ts | 6 +- src/shared/main-process-ndjson-framer.ts | 4 +- 13 files changed, 271 insertions(+), 61 deletions(-) create mode 100644 src/main/claude/structured/claude-structured-session-startup-exit.ts diff --git a/src/main/claude/claude-startup-answer-observation.ts b/src/main/claude/claude-startup-answer-observation.ts index efda9dd91b0a..443a62a21ebb 100644 --- a/src/main/claude/claude-startup-answer-observation.ts +++ b/src/main/claude/claude-startup-answer-observation.ts @@ -2,24 +2,34 @@ import { StringDecoder } from 'node:string_decoder' import type { PipedChildProcess } from '@orca/process-host/process-spec' import { createIncrementalNdjsonFramer } from '../../shared/main-process-ndjson-framer' +// Unescaped quotes occur only as JSON structure, so other records skip this second parse. +const FAILED_RESULT_TOKENS = [/"type"\s*:\s*"result"/, /"is_error"\s*:\s*true/] + +function mayBeFailedResult(text: string): boolean { + return FAILED_RESULT_TOKENS.every((token) => token.test(text)) +} + /** Observe before SDK cleanup can wait on transcript flushing. */ export function claudeStartupAnswerObservation() { let failure: Error | undefined let detach = (): void => {} + const capture = (): void => { + failure ??= Object.assign(new Error('claude reported a failed result during startup'), { + errorClass: 'error_result' + }) + detach() + } const observe = (record: unknown): void => { - if (typeof record !== 'object' || record === null) { - return - } if ( + typeof record === 'object' && + record !== null && 'type' in record && record.type === 'result' && 'is_error' in record && record.is_error === true && !('parent_tool_use_id' in record && record.parent_tool_use_id) ) { - failure ??= Object.assign(new Error('claude reported a failed result during startup'), { - errorClass: 'error_result' - }) + capture() } } return { @@ -27,9 +37,16 @@ export function claudeStartupAnswerObservation() { finish: () => detach(), attach: (child: PipedChildProcess): void => { const decoder = new StringDecoder('utf8') - const framer = createIncrementalNdjsonFramer(observe, () => {}, { - maxLineBytes: Number.POSITIVE_INFINITY - }) + // The shared line budget bounds retention; an oversized failed result still answers from its prefix. + const framer = createIncrementalNdjsonFramer( + observe, + (rejected) => { + if (rejected.kind === 'line-too-long' && mayBeFailedResult(rejected.prefix)) { + capture() + } + }, + { acceptLine: mayBeFailedResult } + ) const onData = (chunk: Buffer | string): void => framer.feed(typeof chunk === 'string' ? chunk : decoder.write(chunk)) detach = () => { diff --git a/src/main/claude/structured/claude-structured-session-acquisition.ts b/src/main/claude/structured/claude-structured-session-acquisition.ts index 691142e6a15c..071d5967cf71 100644 --- a/src/main/claude/structured/claude-structured-session-acquisition.ts +++ b/src/main/claude/structured/claude-structured-session-acquisition.ts @@ -24,10 +24,8 @@ import { } from './claude-structured-session-publication' import type * as sessionState from './claude-structured-session-state' import { resolveClaudeAcquisitionError } from './claude-structured-session-close' -import { - startupAcquisitionRefusal, - withObservedProviderExit -} from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { claudeAcquisitionStartupError } from './claude-structured-session-startup-exit' import { readClaudeTranscriptEntryUuid } from '../claude-transcript-entry-uuid' import { claudeTurnResumePointWriter } from './claude-structured-resume-point' import { withAgentSessionCreatePhase } from '../../observability/agent-session-instrumentation' @@ -296,7 +294,7 @@ export async function acquireClaudeSession({ } catch (error) { unbindReadingControl?.() throw await resolveClaudeAcquisitionError({ - error: startupAcquisitionRefusal(error) ?? attempt.connection?.startupFailure?.() ?? error, + error: await claudeAcquisitionStartupError(error, attempt.connection), sessionId, sessions, attempt, diff --git a/src/main/claude/structured/claude-structured-session-startup-exit.ts b/src/main/claude/structured/claude-structured-session-startup-exit.ts new file mode 100644 index 000000000000..34789c44e67b --- /dev/null +++ b/src/main/claude/structured/claude-structured-session-startup-exit.ts @@ -0,0 +1,38 @@ +import { isClaudeQueryClosed } from '../claude-agent-sdk-control-requests' +import { withTimeout } from '../../../shared/promise-timeout-fallback' +import { + providerExitObserved, + startupAcquisitionRefusal +} from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import type { ClaudeStreamJsonConnection } from '../claude-stream-json-connection' + +/** A physical exit waits briefly for initialize's answer; a delivered answer outranks the exit. */ +export async function claudeStartupAnswerAfterExit( + connection: Pick, + control: Promise, + exit: unknown +): Promise { + const outcome = await withTimeout( + control.then( + () => exit, + (error: unknown) => error + ), + 2_000, + exit + ) + return connection.startupFailure?.() ?? (isClaudeQueryClosed(outcome) ? exit : outcome) +} + +/** An exit before publication can follow initialize's answer, which only publication reads. */ +export async function claudeAcquisitionStartupError( + error: unknown, + connection: ClaudeStreamJsonConnection | null +): Promise { + const refusal = startupAcquisitionRefusal(error) + if (refusal || !connection) { + return refusal ?? error + } + return providerExitObserved(error) + ? claudeStartupAnswerAfterExit(connection, connection.initializationResult(), error) + : (connection.startupFailure?.() ?? error) +} diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index 22deb16a4108..1d579a129d2a 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -6,11 +6,15 @@ import { runClaudeControl } from '../claude-agent-sdk-control-requests' import { retryableStructuredAgentSessionStart } from '../../native-chat/agent-session-wire/structured-agent-session-start-retry' import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import { + PROVIDER_SESSION_ID, adapterAtPublishFor, fakeClaude, identityFor } from './claude-structured-session-test-support' import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' +import { ClaudeStructuredSessionAdapter } from './claude-structured-session-adapter' +import { retryableStructuredAgentSessionAcquisition } from '../../native-chat/agent-session-wire/structured-agent-session-start-retry' +import { NDJSON_MAX_LINE_BYTES } from '../../../shared/main-process-ndjson-framer' afterEach(() => vi.useRealTimers()) @@ -137,3 +141,86 @@ it('settles an unanswered Claude startup exit with the same bounded recovery eli claude.dispose() await adapter.closeAll() }) + +it('keeps a rejected initialize final when the child exits before acquisition publishes', async () => { + const claude = fakeClaude({ holdInit: true }) + const probing = Promise.withResolvers() + const startTime = Promise.withResolvers() + const adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: '/work/repo', + claudeConfigDir: '/accounts/claude', + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: null, + resumesTranscript: false, + cliFlag: async () => 'supported' as const, + continuesChain: false + }), + openConnection: claude.openConnection, + readProcessStartTime: async () => { + probing.resolve() + return startTime.promise + } + }) + const result = adapter + .acquire({ identity: identityFor(), fence: 7, spawnToken: 'prepublication' }) + .catch((error) => error) + await probing.promise + const connection = claude.connections[0] + // The SDK's answer to an error control_response for initialize, as the published path sees it. + const answer = runClaudeControl( + 'initialize', + () => + Promise.reject( + Object.assign(new Error('initialize rejected'), { errorClass: 'control_request_failed' }) + ), + null + ) + connection.initializationResult = () => answer + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'unverifiable' } + connection.close = async () => false + connection.handlers.onExit?.(withObservedProviderExit(new Error('claude exited (code 1)')), { + expected: false + }) + startTime.resolve(1_700_000_000_000) + const error = await result + expect(error.cause.cause).toMatchObject({ errorClass: 'control_request_failed' }) + expect(retryableStructuredAgentSessionAcquisition(error)).toBe(false) + claude.dispose() + connection.close = async () => true + await adapter.closeAll() +}) + +it('parses only a failed root result at startup and bounds what it retains', async () => { + const child = createFakePipedChild() + const observation = claudeStartupAnswerObservation() + observation.attach(child) + const parse = vi.spyOn(JSON, 'parse') + const hook = (output: string) => + `${JSON.stringify({ + type: 'system', + subtype: 'hook_response', + hook_name: 'SessionStart:startup', + hook_event: 'SessionStart', + output, + stdout: '', + stderr: '' + })}\n` + const oversized = hook('x'.repeat(NDJSON_MAX_LINE_BYTES + 1)) + child.stdout.write(oversized.slice(0, NDJSON_MAX_LINE_BYTES / 2)) + child.stdout.write(oversized.slice(NDJSON_MAX_LINE_BYTES / 2)) + child.stdout.write(hook(JSON.stringify({ type: 'result', is_error: true }))) + child.stdout.write( + `${JSON.stringify({ type: 'result', subtype: 'error_during_execution', is_error: true })}\n` + ) + await new Promise((resolve) => setImmediate(resolve)) + const parsed = parse.mock.calls.map(([text]) => String(text)) + parse.mockRestore() + expect(parsed.filter((text) => text.includes('hook_response'))).toEqual([]) + expect(parsed.filter((text) => text.includes('error_during_execution'))).toHaveLength(1) + expect(observation.failure()).toMatchObject({ errorClass: 'error_result' }) + expect(child.stdout.listenerCount('data')).toBe(0) +}) diff --git a/src/main/claude/structured/claude-structured-session-startup.ts b/src/main/claude/structured/claude-structured-session-startup.ts index 0b67558fd3f9..0f333f09ce78 100644 --- a/src/main/claude/structured/claude-structured-session-startup.ts +++ b/src/main/claude/structured/claude-structured-session-startup.ts @@ -1,5 +1,4 @@ import { isClaudeQueryClosed } from '../claude-agent-sdk-control-requests' -import { withTimeout } from '../../../shared/promise-timeout-fallback' import { startupAcquisitionRefusal, providerExitObserved @@ -32,6 +31,7 @@ import { readClaudeSettingsEffort } from './launch/claude-structured-session-options' import { claudeExecutionObservations } from './claude-structured-execution-observations' +import { claudeStartupAnswerAfterExit } from './claude-structured-session-startup-exit' import type { ClaudeAuthDiagnostic, ClaudeSession, @@ -100,18 +100,7 @@ export async function readClaudeStartupFacts(input: { if (startupAcquisitionRefusal(error) || !providerExitObserved(error)) { throw error } - const outcome = await withTimeout( - control.then( - () => ({ error }), - (error: unknown) => ({ error }) - ), - 2_000, - { error } - ) - throw ( - input.connection.startupFailure?.() ?? - (isClaudeQueryClosed(outcome.error) ? error : outcome.error) - ) + throw await claudeStartupAnswerAfterExit(input.connection, control, error) }), new Promise((_resolve, reject) => { timer = setTimeout( diff --git a/src/main/codex/app-server/codex-startup-failure.ts b/src/main/codex/app-server/codex-startup-failure.ts index ecdf33548413..ed67510a5913 100644 --- a/src/main/codex/app-server/codex-startup-failure.ts +++ b/src/main/codex/app-server/codex-startup-failure.ts @@ -24,7 +24,6 @@ export function temporaryCodexStartupAnswer(error: CodexAppServerRequestError): return true } if ( - error.method === 'thread/start' && 'reason' in data && data.reason === 'cloudConfigBundle' && !('action' in data && data.action === 'relogin') && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts index e2b344d2b619..1828de1798a8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts @@ -1,4 +1,5 @@ import { providerStartupFailureFact } from './structured-agent-session-failure-text' +import { sameProviderChild } from './structured-agent-session-provider-child' import { retryableStructuredAgentSessionStart } from './structured-agent-session-start-retry' import { waitForPromiseWithSignal } from '../../../shared/abort-signal-reason' import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' @@ -23,8 +24,16 @@ export async function openStructuredAgentSessionProviderChild( if (!context.runtimeState.startupAttempts.onClock(sessionId, child)) { return } - // Physical exit ends readiness; it must not erase the outstanding protocol answer. - const attempt = context.runtimeState.acquireAborts.begin(sessionId) + // A start flight joins the outstanding answer after exit and retires it; a direct attach has no such owner. + const attempt = context.runtimeState.acquireAborts.begin(sessionId, false, false, () => { + const current = context.sessions.get(sessionId)?.child + return ( + current !== undefined && + current !== null && + sameProviderChild(current, child) && + !context.runtimeState.startFlights.ownsChild(sessionId, child) + ) + }) const observation = context.runtimeState.startupAttempts.observeOpen( sessionId, child, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts index a0011bfd5046..1d3d7f565409 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts @@ -20,6 +20,7 @@ import { noteStructuredAgentSessionReportedOptions } from './structured-agent-session-ran-options' import type { StructuredAgentSessionAcquireAborts } from './structured-agent-session-acquire-aborts' +import type { StructuredAgentSessionStartFlights } from './structured-agent-session-start-flight' import { recordAgentSessionProviderHandle } from '../../runtime/agent-session/ownership/agent-session-provider-handle-transition' import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' import { settleUnheldStructuredAgentSessionTurns } from './structured-agent-session-unheld-turns' @@ -36,7 +37,11 @@ export type StructuredAgentSessionProviderStartedContext = { now: () => number publishStatus?: (sessionId: string) => void runtimeState: { - startupAttempts: Pick + startupAttempts: Pick< + StructuredAgentSessionStartupAttempts, + 'ready' | 'onClock' | 'readiness' | 'observeOpen' + > + startFlights: Pick acquireAborts: Pick } /** Commits the child's buffered lifecycle events. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts index 4fb0f03c6917..802a46f2ee43 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts @@ -30,6 +30,7 @@ import { } from './structured-agent-session-failure-text' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' import { sameProviderChild } from './structured-agent-session-provider-child' +import { providerRequestTimeoutOf } from '../../provider-process/provider-request-timeout' export function registerStructuredAgentSessionStart( context: StructuredAgentSessionAttachContext, @@ -84,19 +85,30 @@ export function registerStructuredAgentSessionStart( if (ready === 'ready') { return result } - if ( - handle.result.opening && + const opening = handle.result.opening + const join = + opening !== undefined && handle.result.retry !== false && !finalStartupExitFailure(context.sessions.get(sessionId)?.lastEndedChild?.failure) - ) { + if (join) { + // The ended child disarmed the ceiling timer; the join must not outlast it. await lock.wait( waitForPromiseWithSignal( - withTimeout(handle.result.opening, 2_000, undefined), + withTimeout( + opening, + Math.max(0, Math.min(2_000, attempts.untilCeiling(registeredAt))), + undefined + ), acquire.signal ) ) - handle.result.cancelOpen?.() - throwIfSignalAborted(acquire.signal) + } + // The flight owns this open after exit, whether or not its answer was awaited. + handle.result.cancelOpen?.() + throwIfSignalAborted(acquire.signal) + if (join && attempts.pastCeiling(registeredAt)) { + result = { ...failedStructuredAgentSessionStart(), expired: true } + break } result = await lock.commit(async () => { const session = context.sessions.get(sessionId) @@ -117,8 +129,11 @@ export function registerStructuredAgentSessionStart( } const finalExit = finalStartupExitFailure(ended.failure) const openError = handle.result.openError + // An elapsed provider deadline keeps its words; the later exit doesn't explain it. const openFailure = - ended.cause === 'exit' && openError instanceof Error + ended.cause === 'exit' && + openError instanceof Error && + !providerRequestTimeoutOf(openError) ? providerStartupFailureFact(withObservedProviderExit(openError)) : handle.result.failure const failure = structuredAgentSessionStartFailureFact({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts index d990d91b9276..183c053de0a3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts @@ -240,18 +240,20 @@ describe('positive provider startup retry contracts', () => { -32600, 'thread abc is closing; retry thread/resume after the thread is closed' ), - codex('thread/start', -32600, 'failed to load configuration', { - reason: 'cloudConfigBundle', - errorCode: 'Timeout' - }), - ...[undefined, 503, 403].map((statusCode) => - codex('thread/start', -32600, 'failed to load configuration', { + ...['thread/start', 'thread/resume'].flatMap((method) => [ + codex(method, -32600, 'failed to load configuration', { reason: 'cloudConfigBundle', - errorCode: 'RequestFailed', - statusCode, - detail: 'Failed to load cloud config bundle (workspace-managed policies).' - }) - ), + errorCode: 'Timeout' + }), + ...[undefined, 503, 403].map((statusCode) => + codex(method, -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode: 'RequestFailed', + statusCode, + detail: 'Failed to load cloud config bundle (workspace-managed policies).' + }) + ) + ]), codex( 'thread/start', -32603, @@ -262,17 +264,19 @@ describe('positive provider startup retry contracts', () => { ])('retries only after separately proving absence: $message', expectExitEvidenceRequired) it.each([ - ...['Auth', 'InvalidBundle', 'Internal', 'RequestFailed'].map((errorCode) => - codex('thread/start', -32600, 'failed to load configuration', { + ...['thread/start', 'thread/resume'].flatMap((method) => [ + ...['Auth', 'InvalidBundle', 'Internal', 'RequestFailed'].map((errorCode) => + codex(method, -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode + }) + ), + codex(method, -32600, 'failed to load configuration', { reason: 'cloudConfigBundle', - errorCode + errorCode: 'Timeout', + action: 'relogin' }) - ), - codex('thread/start', -32600, 'failed to load configuration', { - reason: 'cloudConfigBundle', - errorCode: 'Timeout', - action: 'relogin' - }), + ]), codex('thread/start', -32603, 'error creating thread: outcome unknown'), codex('thread/start', -32603, 'error creating thread: internal error'), codex('thread/resume', -32600, 'please retry this unknown answer') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts index 071586513627..4ccc86e02ffc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts @@ -15,6 +15,7 @@ import { projectStructuredAgentSessionMessages } from '../../../shared/agent-ses import { HOST_TEST_SESSION as SESSION, HOST_TEST_THREAD as THREAD, + hostTestAttachParams, hostTestOperationId } from './structured-agent-session-host-test-data' import type { StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' @@ -356,7 +357,11 @@ describe('one logical start across physical attempts', () => { 'keeps the typed %s exit-only refusal final even when raw open never settles', async (kind) => { await restartWith(publishFirst) - providerOpen.mockImplementationOnce(() => new Promise(() => {})) + let signal: AbortSignal | undefined + providerOpen.mockImplementationOnce((input) => { + signal = input.signal + return new Promise(() => {}) + }) const id = await heldBehindStart('one request') const child = host.collaboratorsForTests().sessions.get(SESSION)!.child! await host.handleAdapterEvent({ @@ -373,9 +378,38 @@ describe('one logical start across physical attempts', () => { expect((await submission(id))?.rejection).toMatchObject({ kind }) expect(acquire).toHaveBeenCalledTimes(1) expect(dispatch).not.toHaveBeenCalled() + expect(signal?.aborted).toBe(true) } ) + it('retires a direct attach open, which has no start flight, when its child exits', async () => { + await restartWith(publishFirst) + let signal: AbortSignal | undefined + providerOpen.mockImplementationOnce((input) => { + signal = input.signal + return new Promise(() => {}) + }) + const fence = store.getRecord(SESSION)!.lease.runtimeFence + expect(await host.attach(CALLER, hostTestAttachParams(fence))).toMatchObject({ ok: true }) + await eventually(() => expect(signal).toBeDefined()) + expect(host.collaboratorsForTests().runtimeState.startFlights.has(SESSION)).toBe(false) + await endAttempt() + expect(signal?.aborted).toBe(true) + }) + + it('keeps an initialize deadline that rejects after the exit it raced', async () => { + await restartWith(publishFirst) + const open = Promise.withResolvers() + providerOpen.mockReturnValueOnce(open.promise) + const id = await heldBehindStart('deadline after exit') + await endAttempt() + const requestTimeout = { method: 'initialize', timeoutMs: 60_000 } + open.reject(withProviderRequestTimeout(new Error('initialize timed out'), requestTimeout)) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect((await submission(id))?.rejection).toMatchObject({ requestTimeout }) + expect(acquire).toHaveBeenCalledTimes(1) + }) + it('bounds an unanswered open after exit and recovers without waiting for the startup ceiling', async () => { await restartWith(publishFirst) providerOpen.mockImplementationOnce(() => new Promise(() => {})) @@ -574,6 +608,15 @@ describe('one logical start across physical attempts', () => { ) expect(acquire).toHaveBeenCalledTimes(2) expect(await submission(second)).toMatchObject(HOST_STOPPED) + acquire.mockClear() + providerOpen.mockImplementationOnce(() => new Promise(() => {})) + const third = await heldBehindStart('ceiling inside the exit join') + await vi.advanceTimersByTimeAsync(900) + await endAttempt() + await vi.advanceTimersByTimeAsync(150) + await eventually(async () => expect((await submission(third))?.dispatchState).toBe('rejected')) + expect(await submission(third)).toMatchObject(HOST_STOPPED) + expect(acquire).toHaveBeenCalledTimes(1) }) it('keeps an elapsed provider-step deadline final and preserves its words', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts index 99d9620c26e1..35ddc9f85b58 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts @@ -77,7 +77,11 @@ export class StructuredAgentSessionStartupAttempts { } pastCeiling(registeredAt: number): boolean { - return this.now() - registeredAt >= this.limits.ceilingMs + return this.untilCeiling(registeredAt) <= 0 + } + + untilCeiling(registeredAt: number): number { + return registeredAt + this.limits.ceilingMs - this.now() } childFailed( diff --git a/src/shared/main-process-ndjson-framer.ts b/src/shared/main-process-ndjson-framer.ts index 4c579ffebec4..4020486f2dba 100644 --- a/src/shared/main-process-ndjson-framer.ts +++ b/src/shared/main-process-ndjson-framer.ts @@ -37,6 +37,8 @@ export type IncrementalNdjsonFramer = { export type IncrementalNdjsonFramerOptions = { maxLineBytes?: number shouldPause?: () => boolean + /** Skips parsing lines the consumer can rule out from their text. */ + acceptLine?: (line: string) => boolean } function boundedUtf8Prefix(value: string, maxBytes: number): string { @@ -192,7 +194,7 @@ export function createIncrementalNdjsonFramer( lineSegments.push(segment) const line = lineSegments.length === 1 ? lineSegments[0] : lineSegments.join('') clearLine() - if (!/^\s*$/.test(line)) { + if (!/^\s*$/.test(line) && (options.acceptLine?.(line) ?? true)) { let parsed: unknown try { parsed = JSON.parse(line) From 991b57a1f1ca4079d2dd465ede9604727b95f68e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:48:50 -0800 Subject: [PATCH 03/21] Keep saving the last turn when a Claude process exits after startup The exit path skipped saving the resume cursor whenever Orca had not dispatched a message, but turns can settle without an Orca dispatch and an already-started idle session still owns its exit save. Only a start that died before initialize answered and before any turn now skips the save and gets the short cleanup bound. A failed result captured before the first send counts as a startup failure only when no turn ran in that process. --- .../claude-structured-resume-point.ts | 14 +++++++ ...laude-structured-session-exit-lifecycle.ts | 22 +++++++--- ...structured-session-startup-failure.test.ts | 41 ++++++++++++++++++- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/src/main/claude/structured/claude-structured-resume-point.ts b/src/main/claude/structured/claude-structured-resume-point.ts index 3fdb7d5967ae..1d6f01857dee 100644 --- a/src/main/claude/structured/claude-structured-resume-point.ts +++ b/src/main/claude/structured/claude-structured-resume-point.ts @@ -92,6 +92,20 @@ function persistClaudeTurnResumePoint( session.resumePointWrite = write } +/** A dispatch or a turn end since launch; without either, acquisition's saved cursor is still current. */ +export function claudeSessionHasTurnWork( + session: Pick< + ClaudeSession, + 'dispatchSequence' | 'resumePointWrite' | 'turnEndLeafUuid' | 'initialization' + > +): boolean { + return ( + session.dispatchSequence !== 0 || + session.resumePointWrite !== undefined || + session.turnEndLeafUuid !== session.initialization.history.leafUuid + ) +} + /** Drain turn-end writes before close or exit persists the final cursor. */ export async function settledClaudeTurnEndLeaf(session: ClaudeSession): Promise { await session.resumePointWrite?.settled diff --git a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts index b2308de103e9..a31be93750f4 100644 --- a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts @@ -9,7 +9,10 @@ import { startupAcquisitionRefusal } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import { providerSpawnAbsent } from '../../provider-process/provider-executable-missing' -import { settledClaudeTurnEndLeaf } from './claude-structured-resume-point' +import { + claudeSessionHasTurnWork, + settledClaudeTurnEndLeaf +} from './claude-structured-resume-point' import { claudeHeldSends } from '../claude-command-lifecycle' import { claudeCloseReport, settleClaudeExitedSession } from './claude-structured-session-close' import { @@ -49,7 +52,7 @@ export function observeClaudeSessionExit( // Run tree cleanup before recovery; an exit callback proves only root exit. const cleanup = session.connection.close().catch(() => false) const closePromise = - session.dispatchSequence === 0 && providerExitObserved(error) + claudeStartDied(session) && providerExitObserved(error) ? withTimeout(cleanup, 2_000, false) : cleanup const exit: ClaudeSessionExit = { @@ -74,6 +77,12 @@ export function observeClaudeSessionExit( .catch(() => undefined) } +/** Exited before initialize answered and before any turn; turns can settle without an Orca dispatch. */ +function claudeStartDied(session: ClaudeSession): boolean { + // Startup arms later refusals for the published session once initialize facts apply. + return session.initialization.proof.onRefusal === null && !claudeSessionHasTurnWork(session) +} + export function settleClaudeUnexpectedExit( lifecycle: ClaudeExitLifecycle, sessionId: string, @@ -86,8 +95,8 @@ export function settleClaudeUnexpectedExit( settleClaudeExitedSession(exit.session) return } - // Without a dispatch, acquisition's saved cursor is still authoritative. - if (exit.session.dispatchSequence !== 0) { + // A dead start's host open is still waiting on this exit, and nothing advanced acquisition's cursor. + if (!claudeStartDied(exit.session)) { await persistClaudeSessionHandle(sessionId, exit.session, deps).catch((error: unknown) => { console.warn('[claude-resume-point] exit cursor was not persisted:', { sessionId, error }) }) @@ -99,8 +108,9 @@ export function settleClaudeUnexpectedExit( exits.delete(sessionId) const report = claudeCloseReport(exit.connection, true, exit.error) lifecycle.settledExitErrors.set(sessionId, report ?? exit.error) - const startupFailure = - exit.session.dispatchSequence === 0 ? exit.connection.startupFailure?.() : undefined + const startupFailure = claudeSessionHasTurnWork(exit.session) + ? undefined + : exit.connection.startupFailure?.() const ended: ClaudeStructuredSessionEvent = { type: 'ended', sessionId, diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index 1d579a129d2a..7ab8c4fee41f 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -8,8 +8,11 @@ import { withObservedProviderExit } from '../../native-chat/agent-session-wire/s import { PROVIDER_SESSION_ID, adapterAtPublishFor, + adapterFor, fakeClaude, - identityFor + identityFor, + recordingJournalSink, + tick } from './claude-structured-session-test-support' import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' import { ClaudeStructuredSessionAdapter } from './claude-structured-session-adapter' @@ -112,6 +115,42 @@ it('retains a failed result before a stalled SDK transcript flush and never wait await adapter.closeAll() }) +it('treats a turn that settled without an Orca dispatch as past startup on exit', async () => { + const claude = fakeClaude() + const persistedHandles: unknown[] = [] + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterFor(claude, {}, events, persistedHandles) + await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'undispatched', + events: { ...recordingJournalSink(), written: async () => ({ ok: true }) } + }) + const connection = claude.connections[0] + // No send ran, so the startup observer still holds a later failed result. + connection.startupFailure = () => + Object.assign(new Error('claude reported a failed result during startup'), { + errorClass: 'error_result' + }) + const frame = (message: Record) => + connection.handlers.onMessage?.({ session_id: PROVIDER_SESSION_ID, ...message }) + frame({ + type: 'assistant', + uuid: 'undispatched-leaf', + message: { role: 'assistant', content: [{ type: 'text', text: 'answer' }] } + }) + frame({ type: 'result' }) + connection.handlers.onExit?.(withObservedProviderExit(new Error('exited')), { expected: false }) + await tick() + + expect(persistedHandles).toContainEqual( + expect.objectContaining({ leafUuid: 'undispatched-leaf' }) + ) + expect(events.at(-1)).toMatchObject({ type: 'ended', failure: { kind: 'providerExited' } }) + claude.dispose() + await adapter.closeAll() +}) + it('settles an unanswered Claude startup exit with the same bounded recovery eligibility', async () => { vi.useFakeTimers() const claude = fakeClaude({ holdInit: true }) From 557f19e918bfaddd145a0b7ec95fee94dcf2c683 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:57:35 -0800 Subject: [PATCH 04/21] Merge duplicate test imports --- .../claude-structured-session-startup-failure.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index 7ab8c4fee41f..fa147300e246 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -3,7 +3,10 @@ import { afterEach, expect, it, vi } from 'vitest' import { createFakePipedChild } from '../../../shared/__fixtures__/fake-spawned-child' import { claudeStartupAnswerObservation } from '../claude-startup-answer-observation' import { runClaudeControl } from '../claude-agent-sdk-control-requests' -import { retryableStructuredAgentSessionStart } from '../../native-chat/agent-session-wire/structured-agent-session-start-retry' +import { + retryableStructuredAgentSessionAcquisition, + retryableStructuredAgentSessionStart +} from '../../native-chat/agent-session-wire/structured-agent-session-start-retry' import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import { PROVIDER_SESSION_ID, @@ -16,7 +19,6 @@ import { } from './claude-structured-session-test-support' import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' import { ClaudeStructuredSessionAdapter } from './claude-structured-session-adapter' -import { retryableStructuredAgentSessionAcquisition } from '../../native-chat/agent-session-wire/structured-agent-session-start-retry' import { NDJSON_MAX_LINE_BYTES } from '../../../shared/main-process-ndjson-framer' afterEach(() => vi.useRealTimers()) From 8bc59beae4cd38705fe95ba9d387062a62a84485 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:34:16 -0800 Subject: [PATCH 05/21] Bound resumed Claude starts that exit during catch-up and keep untyped refusals final Exit-time cursor writes and the short close bound now key on the host-awaited open actually returning, so a resume that exits while transcript catch-up is pending publishes ended at once instead of waiting on a cursor write. The startup observer no longer reads an answer from an oversized line's prefix. A typed acquisition refusal without a dedicated failure kind keeps its refusal marker, so an exit-only start stays final. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../claude-startup-answer-observation.ts | 25 ++---- ...laude-structured-session-exit-lifecycle.ts | 6 +- .../claude-structured-session-open.ts | 1 + ...structured-session-startup-failure.test.ts | 42 +++++++++ .../claude-structured-session-state.ts | 2 + ...t-session-claude-resume-exit-start.test.ts | 90 +++++++++++++++++++ .../structured-agent-session-failure-text.ts | 4 + ...agent-session-startup-attempt-host.test.ts | 21 +++-- 8 files changed, 166 insertions(+), 25 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts diff --git a/src/main/claude/claude-startup-answer-observation.ts b/src/main/claude/claude-startup-answer-observation.ts index 443a62a21ebb..d1c5093b4098 100644 --- a/src/main/claude/claude-startup-answer-observation.ts +++ b/src/main/claude/claude-startup-answer-observation.ts @@ -13,12 +13,6 @@ function mayBeFailedResult(text: string): boolean { export function claudeStartupAnswerObservation() { let failure: Error | undefined let detach = (): void => {} - const capture = (): void => { - failure ??= Object.assign(new Error('claude reported a failed result during startup'), { - errorClass: 'error_result' - }) - detach() - } const observe = (record: unknown): void => { if ( typeof record === 'object' && @@ -29,7 +23,10 @@ export function claudeStartupAnswerObservation() { record.is_error === true && !('parent_tool_use_id' in record && record.parent_tool_use_id) ) { - capture() + failure ??= Object.assign(new Error('claude reported a failed result during startup'), { + errorClass: 'error_result' + }) + detach() } } return { @@ -37,16 +34,10 @@ export function claudeStartupAnswerObservation() { finish: () => detach(), attach: (child: PipedChildProcess): void => { const decoder = new StringDecoder('utf8') - // The shared line budget bounds retention; an oversized failed result still answers from its prefix. - const framer = createIncrementalNdjsonFramer( - observe, - (rejected) => { - if (rejected.kind === 'line-too-long' && mayBeFailedResult(rejected.prefix)) { - capture() - } - }, - { acceptLine: mayBeFailedResult } - ) + // The shared line budget bounds retention; a rejected prefix cannot prove a root record. + const framer = createIncrementalNdjsonFramer(observe, () => {}, { + acceptLine: mayBeFailedResult + }) const onData = (chunk: Buffer | string): void => framer.feed(typeof chunk === 'string' ? chunk : decoder.write(chunk)) detach = () => { diff --git a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts index a31be93750f4..0bd1870f5180 100644 --- a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts @@ -77,10 +77,10 @@ export function observeClaudeSessionExit( .catch(() => undefined) } -/** Exited before initialize answered and before any turn; turns can settle without an Orca dispatch. */ +/** Exited before the host-awaited open returned and before any turn; turns can settle without an Orca dispatch. */ function claudeStartDied(session: ClaudeSession): boolean { - // Startup arms later refusals for the published session once initialize facts apply. - return session.initialization.proof.onRefusal === null && !claudeSessionHasTurnWork(session) + // Initialize can answer while a resume open still waits on transcript catch-up. + return !session.initialization.opened && !claudeSessionHasTurnWork(session) } export function settleClaudeUnexpectedExit( diff --git a/src/main/claude/structured/claude-structured-session-open.ts b/src/main/claude/structured/claude-structured-session-open.ts index 36c1996bfbbd..eac381d64079 100644 --- a/src/main/claude/structured/claude-structured-session-open.ts +++ b/src/main/claude/structured/claude-structured-session-open.ts @@ -91,5 +91,6 @@ export async function openClaudeStructuredSession(input: { ) current() } + session.initialization.opened = true return { ...result, link: session.link, newConversation: !resumesTranscript } } diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index fa147300e246..94bd1ef74e38 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -265,3 +265,45 @@ it('parses only a failed root result at startup and bounds what it retains', asy expect(observation.failure()).toMatchObject({ errorClass: 'error_result' }) expect(child.stdout.listenerCount('data')).toBe(0) }) + +it('takes no startup answer from a line over the budget', async () => { + const child = createFakePipedChild() + const observation = claudeStartupAnswerObservation() + observation.attach(child) + const flush = () => new Promise((resolve) => setImmediate(resolve)) + const large = 'x'.repeat(NDJSON_MAX_LINE_BYTES + 1) + // Tool input is arbitrary JSON; a result-shaped object inside it is not the provider's answer. + const assistant = `${JSON.stringify({ + type: 'assistant', + parent_tool_use_id: null, + message: { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'tool-1', + name: 'Write', + input: { type: 'result', is_error: true, content: large } + } + ] + } + })}\n` + child.stdout.write(assistant.slice(0, NDJSON_MAX_LINE_BYTES)) + child.stdout.write(assistant.slice(NDJSON_MAX_LINE_BYTES)) + await flush() + expect(observation.failure()).toBeUndefined() + // The parent that marks a subagent result can sit past the retained prefix. + child.stdout.write( + `${JSON.stringify({ type: 'result', is_error: true, errors: [large], parent_tool_use_id: 'task-1' })}\n` + ) + // Intended limit: an oversized failed root result is left to the SDK answer or the unanswered exit. + child.stdout.write(`${JSON.stringify({ type: 'result', is_error: true, errors: [large] })}\n`) + await flush() + expect(observation.failure()).toBeUndefined() + child.stdout.write( + `${JSON.stringify({ type: 'result', subtype: 'error_during_execution', is_error: true })}\n` + ) + await flush() + expect(observation.failure()).toMatchObject({ errorClass: 'error_result' }) + expect(child.stdout.listenerCount('data')).toBe(0) +}) diff --git a/src/main/claude/structured/claude-structured-session-state.ts b/src/main/claude/structured/claude-structured-session-state.ts index e57ee5243fb0..db458874c504 100644 --- a/src/main/claude/structured/claude-structured-session-state.ts +++ b/src/main/claude/structured/claude-structured-session-state.ts @@ -233,6 +233,8 @@ export type ClaudeSession = { cwd: string history: { leafUuid: string | null; covered: boolean; contextActivity: number } onLateRefusal: (error: Error) => void + /** Set only when the host-awaited open returns; each new session starts unopened. */ + opened?: true } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts new file mode 100644 index 000000000000..4a95628dc5fe --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts @@ -0,0 +1,90 @@ +import { expect, it, vi } from 'vitest' +import { ClaudeStructuredSessionAdapter } from '../../claude/structured/claude-structured-session-adapter' +import { + PROVIDER_SESSION_ID, + fakeClaude +} from '../../claude/structured/claude-structured-session-test-support' +import { withObservedProviderExit } from './structured-agent-session-failure-text' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' +import { STRUCTURED_AGENT_SESSION_STARTUP_CEILING_MS } from './structured-agent-session-startup-attempt-contract' +import { + accept, + closeSession, + eventually, + host, + providerOpen, + registerStartupAttemptHostTests, + restartWith, + submission +} from './structured-agent-session-startup-attempt-host-test-support' + +registerStartupAttemptHostTests() + +it('ends a resumed start that exits during transcript catch-up without awaiting a cursor write', async () => { + const claude = fakeClaude() + const catchUp = Promise.withResolvers() + const persisted = Promise.withResolvers() + const persistHandle = vi.fn(() => persisted.promise) + let catchingUp = false + let ended = 0 + const adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: '/work/repo', + claudeConfigDir: '/accounts/claude', + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: 'saved-leaf', + resumesTranscript: true, + continuesChain: true, + cliFlag: async () => 'supported' as const + }), + openConnection: claude.openConnection, + readProcessStartTime: async () => 1_700_000_000_000, + persistHandle, + onEvent: (event) => { + if (event.type === 'ended') { + ended += 1 + void host.handleAdapterEvent(event) + } + } + }) + await restartWith((input) => + adapter.acquire({ + ...input, + events: { + ...input.events, + written: () => { + catchingUp = true + return catchUp.promise + } + } + }) + ) + providerOpen.mockImplementation(adapter.open) + closeSession.mockImplementation(adapter.closeSession) + host.deps.adapter.releaseAcquisition = adapter.releaseAcquisition + const startedAt = Date.now() + const id = await accept('resume queued message') + // Initialize has answered; the open still waits on the journal before catch-up. + await eventually(() => expect(catchingUp).toBe(true)) + const flight = host.collaboratorsForTests().runtimeState.startFlights.get(SESSION)! + const connection = claude.connections[0] + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(withObservedProviderExit(new Error('claude exited (code 1)')), { + expected: false + }) + await vi.advanceTimersByTimeAsync(2_100) + expect(ended).toBe(1) + expect(persistHandle).not.toHaveBeenCalled() + catchUp.resolve({ ok: true }) + persisted.resolve() + await adapter.drainObservedExits() + await flight.outcome + await eventually(async () => expect((await submission(id))?.dispatchState).not.toBe('pending')) + expect(Date.now() - startedAt).toBeLessThan(STRUCTURED_AGENT_SESSION_STARTUP_CEILING_MS) + claude.dispose() + await adapter.closeAll() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts index 333e514b3f3e..25cd8af6d709 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts @@ -150,6 +150,10 @@ export function providerStartupFailureFact(cause?: unknown): SubmissionRejection } const exited = providerExitObserved(cause) return agentSessionFailureFact(exited ? 'providerStartFailed' : 'startFailed', { + // A typed refusal of any reason stays final when only its exit fact reaches the start. + ...(cause instanceof AgentSessionAcquisitionRefusal + ? { refusal: { code: cause.code, details: { reason: cause.reason } } } + : {}), detail: providerDiagnosticOf(cause), argumentProblem: argumentProblemOf(cause), ...(exited ? {} : { requestTimeout: providerRequestTimeoutOf(cause) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts index 4ccc86e02ffc..4e216c1c68ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts @@ -19,6 +19,10 @@ import { hostTestOperationId } from './structured-agent-session-host-test-data' import type { StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' +import { + providerStartupFailureFact, + withObservedProviderExit +} from './structured-agent-session-failure-text' import { CALLER, accept, @@ -353,9 +357,16 @@ describe('one logical start across physical attempts', () => { } ) - it.each(['fullAccessAsRoot', 'historyTooLarge'] as const)( - 'keeps the typed %s exit-only refusal final even when raw open never settles', - async (kind) => { + it.each([ + agentSessionFailureFact('fullAccessAsRoot'), + agentSessionFailureFact('historyTooLarge'), + // A refusal whose reason has no failure kind of its own keeps only its refusal marker. + providerStartupFailureFact( + withObservedProviderExit(new AgentSessionAcquisitionRefusal('typed startup refusal')) + ) + ])( + 'keeps the typed $kind exit-only refusal final even when raw open never settles', + async (failure) => { await restartWith(publishFirst) let signal: AbortSignal | undefined providerOpen.mockImplementationOnce((input) => { @@ -371,11 +382,11 @@ describe('one logical start across physical attempts', () => { acquisitionGeneration: child.generation!, reason: 'root refusal', cause: 'unexpected-exit', - failure: agentSessionFailureFact(kind) + failure }) await vi.advanceTimersByTimeAsync(2_100) await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) - expect((await submission(id))?.rejection).toMatchObject({ kind }) + expect((await submission(id))?.rejection).toMatchObject({ kind: failure.kind }) expect(acquire).toHaveBeenCalledTimes(1) expect(dispatch).not.toHaveBeenCalled() expect(signal?.aborted).toBe(true) From 8773078f72021a2847f46238d1124b6385bf67b7 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:07:00 -0800 Subject: [PATCH 06/21] test: pin the fake clock in the resumed-exit start test Co-Authored-By: Claude Opus 5.5 (1M context) --- ...tructured-agent-session-claude-resume-exit-start.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts index 4a95628dc5fe..7929bf3e83d3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts @@ -21,6 +21,8 @@ import { registerStartupAttemptHostTests() +const STARTED_AT = 1_800_000_000_000 + it('ends a resumed start that exits during transcript catch-up without awaiting a cursor write', async () => { const claude = fakeClaude() const catchUp = Promise.withResolvers() @@ -65,7 +67,7 @@ it('ends a resumed start that exits during transcript catch-up without awaiting providerOpen.mockImplementation(adapter.open) closeSession.mockImplementation(adapter.closeSession) host.deps.adapter.releaseAcquisition = adapter.releaseAcquisition - const startedAt = Date.now() + vi.setSystemTime(STARTED_AT) const id = await accept('resume queued message') // Initialize has answered; the open still waits on the journal before catch-up. await eventually(() => expect(catchingUp).toBe(true)) @@ -84,7 +86,7 @@ it('ends a resumed start that exits during transcript catch-up without awaiting await adapter.drainObservedExits() await flight.outcome await eventually(async () => expect((await submission(id))?.dispatchState).not.toBe('pending')) - expect(Date.now() - startedAt).toBeLessThan(STRUCTURED_AGENT_SESSION_STARTUP_CEILING_MS) + expect(Date.now() - STARTED_AT).toBeLessThan(STRUCTURED_AGENT_SESSION_STARTUP_CEILING_MS) claude.dispose() await adapter.closeAll() }) From e23686c625669f13a6c82cde9988cd7823e27636 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:10:23 -0800 Subject: [PATCH 07/21] test: bound startup answer parsing by candidates at N and 10N records Drive the startup observer through the shared framer at 100 and 1000 irrelevant hook and assistant records, and assert that only the failed root result is parsed, one capture happens, the data listener retires, and no parse follows capture or finish, with identical counts at both populations. Co-Authored-By: Claude Opus 5.5 (1M context) --- ...structured-session-startup-failure.test.ts | 80 ++++++++++++++----- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index 94bd1ef74e38..088308c36df0 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -235,11 +235,8 @@ it('keeps a rejected initialize final when the child exits before acquisition pu await adapter.closeAll() }) -it('parses only a failed root result at startup and bounds what it retains', async () => { - const child = createFakePipedChild() - const observation = claudeStartupAnswerObservation() - observation.attach(child) - const parse = vi.spyOn(JSON, 'parse') +it('parses only failed root results at startup, bounded by candidates rather than records', async () => { + const flush = () => new Promise((resolve) => setImmediate(resolve)) const hook = (output: string) => `${JSON.stringify({ type: 'system', @@ -250,20 +247,65 @@ it('parses only a failed root result at startup and bounds what it retains', asy stdout: '', stderr: '' })}\n` - const oversized = hook('x'.repeat(NDJSON_MAX_LINE_BYTES + 1)) - child.stdout.write(oversized.slice(0, NDJSON_MAX_LINE_BYTES / 2)) - child.stdout.write(oversized.slice(NDJSON_MAX_LINE_BYTES / 2)) - child.stdout.write(hook(JSON.stringify({ type: 'result', is_error: true }))) - child.stdout.write( - `${JSON.stringify({ type: 'result', subtype: 'error_during_execution', is_error: true })}\n` - ) - await new Promise((resolve) => setImmediate(resolve)) - const parsed = parse.mock.calls.map(([text]) => String(text)) - parse.mockRestore() - expect(parsed.filter((text) => text.includes('hook_response'))).toEqual([]) - expect(parsed.filter((text) => text.includes('error_during_execution'))).toHaveLength(1) - expect(observation.failure()).toMatchObject({ errorClass: 'error_result' }) - expect(child.stdout.listenerCount('data')).toBe(0) + const assistant = (text: string) => + `${JSON.stringify({ + type: 'assistant', + parent_tool_use_id: null, + message: { role: 'assistant', content: [{ type: 'text', text }] } + })}\n` + const failedResult = (subtype: string) => + `${JSON.stringify({ type: 'result', subtype, is_error: true })}\n` + // Counts the real observer's work for one child at a given startup record population. + const observeStartup = async (records: number) => { + const child = createFakePipedChild() + const observation = claudeStartupAnswerObservation() + observation.attach(child) + const listenersWhileAttached = child.stdout.listenerCount('data') + const parse = vi.spyOn(JSON, 'parse') + try { + const oversized = hook('x'.repeat(NDJSON_MAX_LINE_BYTES + 1)) + child.stdout.write(oversized.slice(0, NDJSON_MAX_LINE_BYTES / 2)) + child.stdout.write(oversized.slice(NDJSON_MAX_LINE_BYTES / 2)) + for (let index = 0; index < records; index += 1) { + child.stdout.write( + index % 2 === 0 + ? hook(JSON.stringify({ type: 'result', is_error: true, index })) + : assistant(`working ${index}`) + ) + } + child.stdout.write(failedResult('error_during_execution')) + await flush() + const parsed = parse.mock.calls.map(([text]) => String(text)) + const failure = observation.failure() + parse.mockClear() + child.stdout.write(failedResult('error_after_capture')) + child.stdout.write(hook('after capture')) + observation.finish() + child.stdout.write(failedResult('error_after_finish')) + await flush() + return { + listenersWhileAttached, + irrelevantParses: parsed.filter((text) => !text.includes('error_during_execution')).length, + candidateParses: parsed.filter((text) => text.includes('error_during_execution')).length, + failure, + parsesAfterCapture: parse.mock.calls.length, + listenersAfterCapture: child.stdout.listenerCount('data') + } + } finally { + parse.mockRestore() + } + } + const n = await observeStartup(100) + const tenN = await observeStartup(1000) + expect(n).toMatchObject({ + listenersWhileAttached: 1, + irrelevantParses: 0, + candidateParses: 1, + failure: expect.objectContaining({ errorClass: 'error_result' }), + parsesAfterCapture: 0, + listenersAfterCapture: 0 + }) + expect(tenN).toEqual(n) }) it('takes no startup answer from a line over the budget', async () => { From e8b00de471dcddcb709bd706e5eefd9cc574c095 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:57:31 -0800 Subject: [PATCH 08/21] test: shape startup fixtures like the real providers Held opens in the launch test now reject when their child exits, as a real connection does, and the temporary open failures use the provider's actual overload envelope. Also fixes the test-only type errors CI found. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../acp/acp-session-reopen-failure.test.ts | 16 +++++++++++-- ...t-session-claude-resume-exit-start.test.ts | 18 ++++++++++---- ...t-session-codex-input-finalization.spec.ts | 6 ++--- ...agent-session-start-retry-recorded.test.ts | 3 +++ .../rpc/methods/agent-launch-at-rest.test.ts | 24 ++++++++++++++++++- ...hestration-structured-worker-retry.test.ts | 14 +++++++++-- 6 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/main/acp/acp-session-reopen-failure.test.ts b/src/main/acp/acp-session-reopen-failure.test.ts index c86573169515..84ce618fa9ef 100644 --- a/src/main/acp/acp-session-reopen-failure.test.ts +++ b/src/main/acp/acp-session-reopen-failure.test.ts @@ -12,7 +12,12 @@ it.each(['ServerUnavailable', 'CatalogNotReady'])( expect(() => acpReopenTakeover( error, - { key: 'saved-key', sessionId: 'saved-id', mayBeUnsaved: () => false }, + { + key: 'saved-key', + sessionId: 'saved-id', + mayBeUnsaved: () => false, + unannouncedLosses: () => [] + }, { over: false, now: 1, warn } ) ).toThrow(error) @@ -24,7 +29,14 @@ afterEach(closeProviderTimelineRigs) it('propagates a temporary load envelope before takeover or a dead-child assertion can replace it', async () => { const rig = await openAcpAdapterRig({ - launch: { resume: { key: 'saved-key', sessionId: 'saved-id', mayBeUnsaved: () => false } }, + launch: { + resume: { + key: 'saved-key', + sessionId: 'saved-id', + mayBeUnsaved: () => false, + unannouncedLosses: () => [] + } + }, script: (agent) => agent.on('session/load', (frame) => agent.fail(frame, -32603, 'server unavailable', { errorName: 'ServerUnavailable' }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts index 7929bf3e83d3..517aa04d891c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts @@ -4,6 +4,7 @@ import { PROVIDER_SESSION_ID, fakeClaude } from '../../claude/structured/claude-structured-session-test-support' +import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' import { withObservedProviderExit } from './structured-agent-session-failure-text' import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' @@ -30,6 +31,7 @@ it('ends a resumed start that exits during transcript catch-up without awaiting const persistHandle = vi.fn(() => persisted.promise) let catchingUp = false let ended = 0 + const lifecycleTypes: (string | undefined)[] = [] const adapter = new ClaudeStructuredSessionAdapter({ resolveLaunch: async () => ({ pathToClaudeCodeExecutable: 'claude', @@ -48,12 +50,19 @@ it('ends a resumed start that exits during transcript catch-up without awaiting onEvent: (event) => { if (event.type === 'ended') { ended += 1 - void host.handleAdapterEvent(event) + const lifecycle = structuredClaudeLifecycleEvent(event) + lifecycleTypes.push(lifecycle?.type) + if (lifecycle) { + void host.handleAdapterEvent(lifecycle) + } } } }) - await restartWith((input) => - adapter.acquire({ + await restartWith((input) => { + if (!input.events) { + throw new Error('expected the host event sink') + } + return adapter.acquire({ ...input, events: { ...input.events, @@ -63,7 +72,7 @@ it('ends a resumed start that exits during transcript catch-up without awaiting } } }) - ) + }) providerOpen.mockImplementation(adapter.open) closeSession.mockImplementation(adapter.closeSession) host.deps.adapter.releaseAcquisition = adapter.releaseAcquisition @@ -80,6 +89,7 @@ it('ends a resumed start that exits during transcript catch-up without awaiting }) await vi.advanceTimersByTimeAsync(2_100) expect(ended).toBe(1) + expect(lifecycleTypes).toEqual(['ended']) expect(persistHandle).not.toHaveBeenCalled() catchUp.resolve({ ok: true }) persisted.resolve() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts index 480f94bcd167..a82f5299f12e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts @@ -144,9 +144,9 @@ export function registerCodexInputFinalization(current: typeof codexInputFinaliz expect(starts()).toHaveLength(0) throw new CodexAppServerRequestError( 'thread/resume', - -32000, - 'startup failed', - 'startup failed' + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' ) } codex.connections[1]?.handlers.onExit?.(new Error('Codex died before ready'), { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts index 183c053de0a3..158a1aa5605d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts @@ -324,6 +324,9 @@ describe('positive provider startup retry contracts', () => { error: { code: -32600, message: 'Server is draining; retry after reconnecting', data } }) const error = await result + if (!(error instanceof CodexAppServerRequestError)) { + throw new Error('expected a Codex request error') + } expect(error.data).toBe(data) expect(Object.keys(error)).not.toContain('data') expectExitEvidenceRequired(error) diff --git a/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts b/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts index 9b4aa00de645..5c3a4360417b 100644 --- a/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts +++ b/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts @@ -39,7 +39,9 @@ const CLI: Partial = {} const SESSION = 'codex_9b1deb4d_3b7d_4bad_9bdd_2b0d7b3dcb6d' let rig: RestTestRig +const openEnded = new Map void>() beforeEach(async () => { + openEnded.clear() // The launch mints its operation id from the wall clock; the rig's host reads a fixed one. vi.useFakeTimers({ now: HOST_TEST_NOW, toFake: ['Date'] }) rig = await createRestTestRig({ idleSweep: { intervalMs: 60_000 } }) @@ -135,12 +137,16 @@ async function child(sessionId = SESSION) { async function endChild(sessionId = SESSION) { const current = await child(sessionId) + await vi.waitFor(() => expect(openEnded.has(current.acquisitionGeneration)).toBe(true)) await rig.host.handleAdapterEvent({ type: 'ended', ...current, reason: 'exited during initialize', cause: 'unexpected-exit' }) + // A real connection rejects its outstanding open once the child exits. + openEnded.get(current.acquisitionGeneration)?.(new Error('codex app-server connection ended')) + openEnded.delete(current.acquisitionGeneration) await vi.waitFor(() => expect(rig.host.collaboratorsForTests().sessions.get(sessionId)?.child).toBeNull() ) @@ -149,6 +155,17 @@ async function endChild(sessionId = SESSION) { function holdStarts() { const acquire = rig.adapter.acquire.getMockImplementation()! rig.adapter.acquire.mockImplementation(rig.heldStart(acquire)) + const open = rig.host.deps.adapter.open + vi.spyOn(rig.host.deps.adapter, 'open').mockImplementation((input) => + Promise.race([ + open(input), + new Promise((_resolve, reject) => { + if (input.acquisitionGeneration) { + openEnded.set(input.acquisitionGeneration, reject) + } + }) + ]) + ) } it('a whitespace-only submit creates an eager empty chat without sending a turn', async () => { @@ -232,7 +249,12 @@ it.each(['spawn', 'open'] as const)( ) } else { vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValueOnce( - new CodexAppServerRequestError('initialize', -32603, 'temporarily unavailable') + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) ) } const result = await launch(CLI, 'fix the build') diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts index afc0691682c6..aa8254169c7e 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts @@ -114,7 +114,12 @@ it('starts only for its agent-sourced preamble and observes it after a transient expect(rig.adapter.acquire).not.toHaveBeenCalled() expect((await rig.host.journalSnapshot(sessionId)).submissions).toEqual([]) vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValueOnce( - new CodexAppServerRequestError('initialize', -32603, 'temporarily unavailable') + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) ) const result = await deliverPreamble() expect(result.structuredTurnStart).toEqual({ verdict: 'observed' }) @@ -146,7 +151,12 @@ it('reports a preamble as undelivered after startup exhaustion and retires the f const sessionId = created.identity.sessionId expect(rig.store.listVisibleSessionIds()).toContain(sessionId) vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValue( - new CodexAppServerRequestError('initialize', -32603, 'temporarily unavailable') + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) ) await expect(deliverPreamble()).rejects.toMatchObject({ code: 'dispatch_preamble_undelivered' }) expect(rig.adapter.acquire).toHaveBeenCalledTimes(5) From f21eac9f0531b997ac17de475a9118d11622738a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:00:00 -0800 Subject: [PATCH 09/21] test: use the real Codex canonical-filename refusal wire capture Co-Authored-By: Claude Opus 5.5 (1M context) --- .../fixtures/start-resume-canonical-filename-refusal.json | 4 ++-- .../structured-agent-session-start-retry-recorded.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json b/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json index f04ec730b360..ac7884786668 100644 --- a/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json +++ b/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json @@ -1,6 +1,6 @@ { - "capture": "Native resume failure logged on 2026-10-30; only the account path is sanitized. The original dispatcher did not retain error.data, so it is unknown here.", + "capture": "Wire response from codex-cli 0.166.0 thread/resume captured in an isolated QA home on 2026-11-02; paths sanitized; the response has no error.data", "method": "thread/resume", "code": -32600, - "message": "paginated rollout path `/qa-home/.codex/sessions/rollout-2026-10-28T18-52-57.454Z-01a183db-b8d6-7170-af39-e004651c1e1e.jsonl` does not have a canonical rollout filename" + "message": "paginated rollout path `/qa-home/userdata/codex-runtime-home/home/sessions/2026/11/01/rollout-2026-11-01T20-18-12.123Z-01a19a7a-a7b8-7790-9e1d-73d9040a7f14.jsonl` does not have a canonical rollout filename" } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts index 158a1aa5605d..555d8c51b6b0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts @@ -200,7 +200,7 @@ describe('recorded command rejection wrappers at the startup classifier', () => }) }) -// Source-shaped declarations: these provider-side temporary conditions are not locally reproducible. +// The wire capture has no error.data, so the error is built without one. it('keeps the recorded canonical rollout filename refusal final on the first answer', () => { const { method, code, message } = canonicalFilenameRefusal expectFinalWithExitEvidence(new CodexAppServerRequestError(method, code, message, message)) From 3950a7f9304d1bfd5d6a6e5dc64bbd266b4ea036 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:14:12 -0800 Subject: [PATCH 10/21] test(claude): align merged startup tests with answer-wins and final generic results An exit with no stderr leaves the start to Claude's buffered result, so that result is the start's one failure. A generic startup failure result is final after one start; only an unanswered exit retries, and the recovery case now recovers from such an exit. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../claude-stream-json-exit-error.test.ts | 4 +- .../claude-structured-session-startup.test.ts | 9 +++-- ...nt-session-startup-provider-errors.test.ts | 37 +++++++++++-------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/src/main/claude/claude-stream-json-exit-error.test.ts b/src/main/claude/claude-stream-json-exit-error.test.ts index 0e36717ba7ff..0b1ccf418dfb 100644 --- a/src/main/claude/claude-stream-json-exit-error.test.ts +++ b/src/main/claude/claude-stream-json-exit-error.test.ts @@ -24,7 +24,7 @@ it('treats the recorded missing-conversation exit as an explained final refusal' expect(retryableStructuredAgentSessionStart(error)).toBe(false) }) -it('reads the same refusal from the recorded result, and leaves other startup results retryable', () => { +it('reads the same refusal from the recorded result, and makes other startup results final', () => { const refusal = claudeStartupResultError(frame) expect(providerStartupFailureFact(refusal)).toEqual({ kind: 'savedConversationMissing', @@ -36,5 +36,5 @@ it('reads the same refusal from the recorded result, and leaves other startup re kind: 'startFailed', detail: { text: 'managed settings failed', audience: 'log' } }) - expect(retryableStructuredAgentSessionStart(generic)).toBe(true) + expect(retryableStructuredAgentSessionStart(generic)).toBe(false) }) diff --git a/src/main/claude/structured/claude-structured-session-startup.test.ts b/src/main/claude/structured/claude-structured-session-startup.test.ts index 8136ddeef332..67c7a9e7569a 100644 --- a/src/main/claude/structured/claude-structured-session-startup.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup.test.ts @@ -197,7 +197,7 @@ describe('Claude host-awaited open', () => { // Recorded result; the orderings are constructed. A start another failure ended reports that cause. it.each([ ['initialize rejected', 1], - ['an exit with no stderr', 1], + ['an exit with no stderr', 0], ['an exit naming the missing conversation', 0], ['the same result', 0] ] as const)('a startup failure result after %s keeps %i rows', async (ended, rows) => { @@ -220,9 +220,12 @@ describe('Claude host-awaited open', () => { claudeStreamJsonExitError(stderr, { code: 1, signal: null, processless: false }) ) } - // An exit declines at once, before the buffered result drains. + // An exit naming no cause leaves the start to the buffered result, so that result is its one failure. claudeFrame(connection, { ...MISSING, session_id: PROVIDER_SESSION_ID }) - await open + const error = await open + if (ended === 'an exit with no stderr') { + expect(startupAcquisitionRefusal(error)?.reason).toBe('savedConversationMissing') + } expect(errorRows(appendItem)).toHaveLength(rows) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-provider-errors.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-provider-errors.test.ts index 751c46678b8b..b100e4a05ef1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-provider-errors.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-provider-errors.test.ts @@ -208,19 +208,21 @@ it.each(['result', 'stderr exit', 'result then exit', 'stderr exit then result'] ) // Real order: the result lands before the exit, so its own error decides the attempt. +// Claude's answer is final; only an exit with no answer retries. it.each([ - ['result', 'startFailed', "Claude couldn't start. Send your message to try again."], - ['result then exit', 'startFailed', "Claude couldn't start. Send your message to try again."], + ['result', 1, 'startFailed', "Claude couldn't start. Send your message to try again."], + ['result then exit', 1, 'startFailed', "Claude couldn't start. Send your message to try again."], [ 'stderr exit', + 5, 'providerStartFailed', 'Claude stopped before it finished starting. Send your message to try again.' ] ] as const)( - 'retries a generic startup failure (%s) up to five times, then shows one row', - async (order, kind, text) => { + 'ends a generic startup failure (%s) after %i start(s) with one row', + async (order, starts, kind, text) => { const id = await send('continue this chat') - for (let attempt = 1; attempt <= 5; attempt += 1) { + for (let attempt = 1; attempt <= starts; attempt += 1) { const connection = await starting(attempt) if (order !== 'stderr exit') { frame(connection, GENERIC) @@ -234,7 +236,7 @@ it.each([ } } const errors = await settled(id) - expect(claude.connections).toHaveLength(5) + expect(claude.connections).toHaveLength(starts) if (order === 'result') { // The host stops each failed child; the result itself is no exit evidence. for (const connection of claude.connections) { @@ -247,6 +249,17 @@ it.each([ text, failure: { kind, detail: { text: expect.stringContaining(GENERIC_TEXT), audience: 'log' } } }) + if (order !== 'stderr exit') { + // The ended start logs what Claude said. + expect(log.entries).toContainEqual( + expect.objectContaining({ + message: 'a provider could not finish starting', + fields: expect.objectContaining({ + error: expect.objectContaining({ message: expect.stringContaining(GENERIC_TEXT) }) + }) + }) + ) + } } ) @@ -271,9 +284,9 @@ it('leaves a Stop during startup to Stop when Claude then reports a startup fail expect(claude.connections).toHaveLength(1) }) -it('recovers from a generic startup failure with no failure row, logging what Claude said', async () => { +it('recovers from an unanswered startup exit with no failure row', async () => { await send('continue this chat') - frame(await starting(1), GENERIC) + exit(await starting(1), GENERIC_TEXT) const recovered = await starting(2) recovered.initialization.resolve({}) // The queued send reaches the recovered Claude. @@ -283,12 +296,4 @@ it('recovers from a generic startup failure with no failure row, logging what Cl expect(items.filter((item) => item.body.kind === 'status' && item.body.tone === 'error')).toEqual( [] ) - expect(log.entries).toContainEqual( - expect.objectContaining({ - message: 'a provider could not finish starting', - fields: expect.objectContaining({ - error: expect.objectContaining({ message: expect.stringContaining(GENERIC_TEXT) }) - }) - }) - ) }) From 69aac89dfb40d0bc800a3d87d914227008e356e3 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:50:27 -0800 Subject: [PATCH 11/21] fix(claude): let a final initialize answer outrank a physical exit reported fault-first The connection reports one physical root exit through onFault and then onExit. It now marks that error as an observed exit before either callback, and acquisition's onFault leaves a proven exit to onExit's bounded initialize join, so a final initialize rejection that arrives just after the exit owns the start instead of a retryable bare exit. Typed refusals, live-root faults and processless spawn faults still end the start at once. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../claude-stream-json-connection.test.ts | 82 ++++++++++++++++++- .../claude/claude-stream-json-connection.ts | 7 +- .../claude-structured-session-acquisition.ts | 4 +- .../claude-structured-startup-outcome.ts | 10 ++- 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index ad18841ffbd4..7092556ad238 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -1,7 +1,7 @@ import { execFileSync } from 'node:child_process' import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { dirname, join } from 'node:path' import { providerDiagnosticOf } from '../../shared/agent-session/failures/agent-session-failure' import { afterEach, describe, expect, it, vi } from 'vitest' import { spawnProcess } from '@orca/process-host' @@ -24,6 +24,14 @@ import { import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { claudeProviderHandle } from '../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' import { providerExecutableMissing } from '../provider-process/provider-executable-missing' +import { findInErrorChain } from '../provider-process/provider-error-chain' +import { providerExitObserved } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { retryableStructuredAgentSessionStart } from '../native-chat/agent-session-wire/structured-agent-session-start-retry' +import { + adapterAtPublishFor, + fakeClaude, + identityFor +} from './structured/claude-structured-session-test-support' // Exercise the real SDK against a scripted child CLI. const FAKE_CLI = join(__dirname, '__fixtures__', 'claude-agent-sdk-scripted-cli.mjs') @@ -502,6 +510,78 @@ describe('Claude stream-json connection', () => { } }) + it('lets a final initialize answer after a physical exit own the start despite fault-first callbacks', async () => { + const go = join(mkdtempSync(join(tmpdir(), 'claude-sdk-exit-answer-')), 'go') + scratchDirs.push(dirname(go)) + const scenario = scriptScenario([ + { awaitFile: { path: go, timeoutMs: 15_000 } }, + { delayMs: 20 }, + { exit: 1 } + ]) + // Real SDK and child; only initialize's answer is held so it can arrive after the exit. + const { query: sdkQuery } = await import('@anthropic-ai/claude-agent-sdk') + const answer = Promise.withResolvers() + const queryImpl = ((args: Parameters[0]) => + Object.assign(sdkQuery(args), { initializationResult: () => answer.promise })) as typeof query + const reports: string[] = [] + const exited = Promise.withResolvers() + const claude = fakeClaude() + const adapter = adapterAtPublishFor({ + ...claude, + openConnection: (_launch, handlers = {}) => + open( + launchFor(scenario), + { + ...handlers, + onFault: (error) => { + reports.push(providerExitObserved(error) ? 'exit-fault' : 'fault') + handlers.onFault?.(error) + }, + onExit: (error, exit) => { + reports.push('exit') + handlers.onExit?.(error, exit) + exited.resolve() + } + }, + queryImpl + ) + }) + try { + const acquired = await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 's' }) + let settled = false + const opened = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .then( + () => null, + (error: unknown) => error + ) + .finally(() => { + settled = true + }) + writeFileSync(go, '') + await exited.promise + await new Promise((resolve) => setTimeout(resolve, 30)) + + expect(reports).toEqual(['exit-fault', 'exit']) + expect(settled).toBe(false) + answer.reject( + Object.assign(new Error('initialize failed'), { errorClass: 'control_request_failed' }) + ) + const failure = await opened + expect( + findInErrorChain(failure, (inner) => (inner as { errorClass?: string }).errorClass) + ).toBe('control_request_failed') + expect(retryableStructuredAgentSessionStart(failure)).toBe(false) + } finally { + await adapter.closeAll() + } + }, 20_000) + it.runIf(process.platform !== 'win32')( 'proves a natural SDK exit and cleans up its descendant before recovery', async () => { diff --git a/src/main/claude/claude-stream-json-connection.ts b/src/main/claude/claude-stream-json-connection.ts index 195548c29e7e..83ed271c0d91 100644 --- a/src/main/claude/claude-stream-json-connection.ts +++ b/src/main/claude/claude-stream-json-connection.ts @@ -23,6 +23,7 @@ import { import type { ClaudeStructuredSdkOptions } from './structured/launch/claude-structured-launch-resolution' import { withProviderSpawnAbsence } from '../provider-process/provider-executable-missing' import { claudeStreamJsonExitError } from './claude-stream-json-exit-error' +import { withObservedProviderExit } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' export { ClaudeControlRequestError } @@ -66,7 +67,7 @@ export type ClaudeStreamJsonConnectionHandlers = { canUseTool?: CanUseTool /** `request_user_dialog` control; the CLI only emits kinds declared in `supportedDialogKinds`. */ onUserDialog?: OnUserDialog - /** A transport/process fault that is not itself first-hand root exit proof. */ + /** A transport/process fault; a physical root exit arrives marked observed and is followed by onExit. */ onFault?: (error: Error) => void /** Report observed root exit once; expected stays true even after an unproven close attempt. */ onExit?: (error: Error, exit?: { expected: boolean }) => void @@ -183,6 +184,10 @@ export async function openClaudeStreamJsonConnection( const error = claudeStreamJsonExitError(managed.stderrTail(), managed.exitStatus, cause) terminalError = withProviderSpawnAbsence(error, managed.spawnAbsence) } + // One physical exit reaches both callbacks; mark it first so onFault cannot treat it as live-root. + if (managed.rootExitObserved && !closing) { + withObservedProviderExit(terminalError) + } inbox.fail(terminalError) if (!closing && !faultReported) { faultReported = true diff --git a/src/main/claude/structured/claude-structured-session-acquisition.ts b/src/main/claude/structured/claude-structured-session-acquisition.ts index 42c593d433f7..1cf291ea36db 100644 --- a/src/main/claude/structured/claude-structured-session-acquisition.ts +++ b/src/main/claude/structured/claude-structured-session-acquisition.ts @@ -34,7 +34,7 @@ import { createClaudeSessionJournalTranslator } from './journal/claude-structured-session-journal-control' import { isRootClaudeFrame } from '../claude-turn-opening' -import { createClaudeStartupOutcome } from './claude-structured-startup-outcome' +import { createClaudeStartupOutcome, endsClaudeStartup } from './claude-structured-startup-outcome' import { claudeStartupResultError } from '../claude-stream-json-exit-error' export async function acquireClaudeSession({ @@ -64,7 +64,7 @@ export async function acquireClaudeSession({ const startup = createClaudeStartupOutcome() const childFailed = (error: Error, ended?: Promise): void => { childEnded ??= error - void (ended ? ended.then(startup.fail) : startup.fail(error)) + void (ended ? ended.then(startup.fail) : endsClaudeStartup(error) && startup.fail(error)) initProof.reject(error) } const translator = createClaudeSessionJournalTranslator( diff --git a/src/main/claude/structured/claude-structured-startup-outcome.ts b/src/main/claude/structured/claude-structured-startup-outcome.ts index 335eab55cb84..3a73f80d2320 100644 --- a/src/main/claude/structured/claude-structured-startup-outcome.ts +++ b/src/main/claude/structured/claude-structured-startup-outcome.ts @@ -1,4 +1,7 @@ -import { startupAcquisitionRefusal } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { + providerExitObserved, + startupAcquisitionRefusal +} from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' /** One acquisition's startup: its first ending cause is the one its open rejects with. */ export type ClaudeStartupOutcome = { @@ -14,6 +17,11 @@ export type ClaudeStartupOutcome = { opened: () => boolean } +/** A fault ends the start now unless it is a proven physical exit, which onExit's initialize join settles. */ +export function endsClaudeStartup(error: unknown): boolean { + return startupAcquisitionRefusal(error) !== undefined || !providerExitObserved(error) +} + export function createClaudeStartupOutcome(): ClaudeStartupOutcome { let state: 'pending' | 'opened' | { accepted: Error } | { failed: unknown } = 'pending' let rejectFailed = (_error: unknown): void => {} From 625e0c5d66f74c3bbfe3323bec9fa6de5cc29871 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:09:37 -0800 Subject: [PATCH 12/21] test(claude): await a macrotask instead of real time in the exit-answer start test Co-Authored-By: Claude Opus 5.5 (1M context) --- src/main/claude/claude-stream-json-connection.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index 7092556ad238..98d96e41fe51 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -565,7 +565,8 @@ describe('Claude stream-json connection', () => { }) writeFileSync(go, '') await exited.promise - await new Promise((resolve) => setTimeout(resolve, 30)) + // A fault-first start would already have settled; one macrotask drains its rejection chain. + await new Promise((resolve) => setImmediate(resolve)) expect(reports).toEqual(['exit-fault', 'exit']) expect(settled).toBe(false) @@ -580,7 +581,7 @@ describe('Claude stream-json connection', () => { } finally { await adapter.closeAll() } - }, 20_000) + }) it.runIf(process.platform !== 'win32')( 'proves a natural SDK exit and cleans up its descendant before recovery', From eebfbc6b3418bf2e9076dacf9c427323ea2a7658 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:16:09 -0800 Subject: [PATCH 13/21] test(claude): replace casts in the exit-answer start test with an annotation and guard Co-Authored-By: Claude Opus 5.5 (1M context) --- src/main/claude/claude-stream-json-connection.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index 98d96e41fe51..737a55c6a78c 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -521,8 +521,8 @@ describe('Claude stream-json connection', () => { // Real SDK and child; only initialize's answer is held so it can arrive after the exit. const { query: sdkQuery } = await import('@anthropic-ai/claude-agent-sdk') const answer = Promise.withResolvers() - const queryImpl = ((args: Parameters[0]) => - Object.assign(sdkQuery(args), { initializationResult: () => answer.promise })) as typeof query + const queryImpl: typeof query = (args) => + Object.assign(sdkQuery(args), { initializationResult: () => answer.promise }) const reports: string[] = [] const exited = Promise.withResolvers() const claude = fakeClaude() @@ -575,7 +575,7 @@ describe('Claude stream-json connection', () => { ) const failure = await opened expect( - findInErrorChain(failure, (inner) => (inner as { errorClass?: string }).errorClass) + findInErrorChain(failure, (inner) => ('errorClass' in inner ? inner.errorClass : undefined)) ).toBe('control_request_failed') expect(retryableStructuredAgentSessionStart(failure)).toBe(false) } finally { From 97042931a9f63766773b29d96624d9589b222a0b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 3 Nov 2026 02:07:10 -0800 Subject: [PATCH 14/21] fix(agent-session): let each adapter decide a failed start, and give the start one owner A start that the agent refused could still be replayed: the host joined a child's exit with its open's error and rebuilt the meaning from provider error classes, and an answer written just before exit could lose to the exit itself. Each adapter now answers its own acquire, open and required startup reads with one typed result (final, or safe to run once more), keeping a final answer beneath the exit proofs cleanup adds. Transports keep reading answers already written after the agent stops reading or exits, within the existing one-second drain, and only a drained exit of the agent's own process counts as unanswered. The child's open is the single owner of the start's outcome: it awaits the adapter's result and required settlement, records the verdict once, writes a direct attach's failure row, and stops the child. The start flight reads that verdict instead of joining exit and open. A start whose adapter gives no outcome within five seconds of its own process exit fails through a timer that no bookkeeping can hold, and the original start deadline stays in force until the start is decided. Tracked in STA-12982. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/main/acp/acp-agent-connection.test.ts | 42 ++- src/main/acp/acp-agent-connection.ts | 65 +++- src/main/acp/acp-errors.ts | 16 + src/main/acp/acp-json-rpc-peer.test.ts | 43 +++ src/main/acp/acp-json-rpc-peer.ts | 92 +++--- src/main/acp/acp-peer-reply.ts | 42 +++ .../acp-scripted-connection.test-fixture.ts | 7 +- .../acp/acp-session-reopen-failure.test.ts | 2 + src/main/acp/acp-session-reopen-failure.ts | 2 +- src/main/acp/acp-session-runtime.ts | 9 +- src/main/acp/acp-startup-failure.ts | 12 - src/main/acp/acp-startup-outcome.ts | 44 +++ .../acp-structured-adapter.test-support.ts | 16 +- .../acp/acp-structured-child-lifecycle.ts | 22 ++ .../acp/acp-structured-session-adapter.ts | 42 +-- src/main/acp/acp-structured-start-open.ts | 45 +++ src/main/claude/claude-startup-failure.ts | 56 ++-- .../claude-stream-json-connection.test.ts | 5 +- .../claude-stream-json-exit-error.test.ts | 5 +- .../claude-structured-session-acquisition.ts | 2 +- .../claude-structured-session-adapter.ts | 47 ++- ...laude-structured-session-exit-lifecycle.ts | 5 + .../claude-structured-session-startup-exit.ts | 8 +- ...structured-session-startup-failure.test.ts | 22 +- .../claude-structured-session-startup.test.ts | 2 +- .../claude-structured-start-ending.ts | 68 ++++ ...x-app-server-connection-exit-order.test.ts | 36 ++ .../app-server/codex-app-server-connection.ts | 23 +- .../app-server/codex-app-server-exit-error.ts | 22 +- ...up-failure.ts => codex-startup-outcome.ts} | 37 ++- .../codex-scripted-child.test-fixture.ts | 7 +- .../codex/codex-structured-session-acquire.ts | 10 + ...odex-structured-session-adapter-fixture.ts | 33 +- .../codex/codex-structured-session-adapter.ts | 18 +- .../codex/codex-structured-session-state.ts | 5 + .../codex/codex-structured-start-verdict.ts | 44 +++ src/main/jsonl-rpc/agent-connection.test.ts | 2 + src/main/jsonl-rpc/agent-connection.ts | 61 ++-- src/main/jsonl-rpc/peer-reply.ts | 70 ++++ src/main/jsonl-rpc/peer.ts | 115 +++---- ...structured-agent-session-adapter-router.ts | 6 + .../structured-agent-session-adapter.ts | 6 + .../structured-agent-session-agent-start.ts | 4 +- ...ured-agent-session-append-delivery.test.ts | 10 +- ...structured-agent-session-attach-failure.ts | 2 +- ...structured-agent-session-attach-publish.ts | 27 +- .../structured-agent-session-child-exit.ts | 16 +- ...red-agent-session-conversation-lifetime.ts | 69 ++-- ...gent-session-dead-generation-settlement.ts | 7 +- ...structured-agent-session-event-recovery.ts | 16 +- .../structured-agent-session-event-sink.ts | 11 +- ...ructured-agent-session-exit-observation.ts | 30 ++ ...d-agent-session-held-start.test-support.ts | 81 ++++- ...ctured-agent-session-host-runtime-state.ts | 15 +- .../structured-agent-session-host.ts | 1 - ...ctured-agent-session-provider-open.test.ts | 33 ++ .../structured-agent-session-provider-open.ts | 155 +++++---- ...t-session-provider-started-outcome.test.ts | 81 +++++ ...ructured-agent-session-provider-started.ts | 309 +++++++----------- .../structured-agent-session-rewind.test.ts | 3 +- ...structured-agent-session-start-attempts.ts | 168 ++++------ ...agent-session-start-failure-writer.test.ts | 10 +- .../structured-agent-session-start-failure.ts | 205 ++++++++++++ ...red-agent-session-start-provider-answer.ts | 142 ++++++++ ...agent-session-start-retry-recorded.test.ts | 26 +- .../structured-agent-session-start-retry.ts | 82 ----- ...gent-session-start-settlement-host.test.ts | 33 +- ...-agent-session-startup-attempt-contract.ts | 16 + ...ssion-startup-attempt-host-test-support.ts | 33 +- ...agent-session-startup-attempt-host.test.ts | 132 +++----- ...tructured-agent-session-startup-attempt.ts | 235 +++++++------ ...ructured-agent-session-startup-tracking.ts | 98 ++++++ src/main/opencode/opencode-acp-connection.ts | 3 +- src/main/pi/pi-scripted-child.test-fixture.ts | 11 +- src/main/pi/rpc-session-adapter.ts | 23 +- src/main/pi/rpc-session-startup.test.ts | 3 +- src/main/pi/rpc-session.ts | 11 +- src/main/pi/rpc-startup-open.ts | 58 ++++ src/main/pi/rpc-startup-outcome.ts | 25 ++ .../provider-exit-output-drain.ts | 39 +++ .../provider-process/provider-output-end.ts | 59 ++++ .../rpc/methods/agent-launch-at-rest.test.ts | 20 +- 82 files changed, 2454 insertions(+), 1064 deletions(-) create mode 100644 src/main/acp/acp-peer-reply.ts delete mode 100644 src/main/acp/acp-startup-failure.ts create mode 100644 src/main/acp/acp-startup-outcome.ts create mode 100644 src/main/acp/acp-structured-start-open.ts create mode 100644 src/main/claude/structured/claude-structured-start-ending.ts rename src/main/codex/app-server/{codex-startup-failure.ts => codex-startup-outcome.ts} (50%) create mode 100644 src/main/codex/codex-structured-start-verdict.ts create mode 100644 src/main/jsonl-rpc/peer-reply.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-provider-started-outcome.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-start-provider-answer.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-tracking.ts create mode 100644 src/main/pi/rpc-startup-open.ts create mode 100644 src/main/pi/rpc-startup-outcome.ts create mode 100644 src/main/provider-process/provider-exit-output-drain.ts create mode 100644 src/main/provider-process/provider-output-end.ts diff --git a/src/main/acp/acp-agent-connection.test.ts b/src/main/acp/acp-agent-connection.test.ts index 7e0bccfb3f69..2659b0080818 100644 --- a/src/main/acp/acp-agent-connection.test.ts +++ b/src/main/acp/acp-agent-connection.test.ts @@ -11,7 +11,9 @@ import { type AcpAgentConnection, type AcpAgentConnectionOptions } from './acp-agent-connection' -import { AcpAuthRequiredError } from './acp-errors' +import { AcpAgentExitedError, AcpAuthRequiredError } from './acp-errors' +import { acpStartFailure } from './acp-startup-outcome' +import captures from '../native-chat/agent-session-wire/__fixtures__/startup-protocol-rejections.json' import { PROVIDER_SPAWN_FAILURE_MARKER } from '../provider-process/provider-spawn-failure-report' import { providerExecutableMissing } from '../provider-process/provider-executable-missing' import { AcpScriptedAgent, deferred, tick } from './acp-scripted-agent.test-support' @@ -236,6 +238,8 @@ describe('ACP process-owning connection', () => { const rejected = expect(connection.prompt([...prompt])).rejects.toThrow('broken pipe') child.stdin.emit('error', new Error('broken pipe')) child.emit('close', 0, null) + // The call waits the drain bound for an answer the agent may already have written. + await vi.advanceTimersByTimeAsync(1_000) await rejected expect(connection.closed).toBe(true) expect(connection.rootVerdict).toBe('live') @@ -307,6 +311,33 @@ describe('ACP process-owning connection', () => { } ) + it('keeps the final answer an agent wrote just before it exited, and only then reads its exit', async () => { + const { response } = captures.acp + const onExit = vi.fn() + const { connection, agent, child } = fixture({ onExit }, { exitOnEnd: false }) + // The recorded refusal answers this start's session request just after the agent exited. + agent.on('session/new', (frame) => { + child.emit('exit', 0, null) + setImmediate(() => agent.send({ ...response, jsonrpc: '2.0', id: frame.id })) + }) + const loaded = connection.start(start).catch((error: unknown) => error) + expect(await loaded).toMatchObject({ code: response.error.code, data: response.error.data }) + expect(acpStartFailure(await loaded).retry).toBe('never') + agent.stdout.end() + await vi.waitFor(() => expect(onExit).toHaveBeenCalledOnce()) + }) + + it('answers an unanswered call with the drained exit, which may start once more', async () => { + const { connection, agent, child } = fixture({}, { exitOnEnd: false }) + agent.on('session/new', () => { + child.emit('exit', 1, null) + agent.stdout.end() + }) + const loaded = connection.start(start).catch((error: unknown) => error) + expect(await loaded).toBeInstanceOf(AcpAgentExitedError) + expect(acpStartFailure(await loaded).retry).toBe('safe') + }) + it('isolates early and late exit subscribers and continues delivering exit', () => { const onDiagnostic = vi.fn(() => { throw new Error('diagnostic') @@ -319,11 +350,10 @@ describe('ACP process-owning connection', () => { connection.onExit(failing) connection.onExit(early) expect(() => child.emit('exit', 0, null)).not.toThrow() - expect(early).toHaveBeenCalledExactlyOnceWith({ - code: 0, - signal: null, - processless: false - }) + expect(early).toHaveBeenCalledExactlyOnceWith( + { code: 0, signal: null, processless: false }, + false + ) expect(() => connection.onExit(failing)).not.toThrow() const late = vi.fn() connection.onExit(late) diff --git a/src/main/acp/acp-agent-connection.ts b/src/main/acp/acp-agent-connection.ts index 516ce945d91c..fb51e37d37ee 100644 --- a/src/main/acp/acp-agent-connection.ts +++ b/src/main/acp/acp-agent-connection.ts @@ -6,11 +6,15 @@ import { type ProviderProcessExit } from '../provider-process/managed-provider-process' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' +import { + afterProviderOutputDrains, + PROVIDER_EXIT_OUTPUT_DRAIN_MS +} from '../provider-process/provider-exit-output-drain' import { withProviderSpawnAbsence, type ProviderSpawnAbsence } from '../provider-process/provider-executable-missing' -import { AcpConnectionClosedError, AcpSignInPromptError } from './acp-errors' +import { AcpAgentExitedError, AcpConnectionClosedError, AcpSignInPromptError } from './acp-errors' import { resolveAcpPeerOptions, type AcpPeerOptions } from './acp-peer-limits' import { AcpSessionRuntime, type AcpSessionRuntimeOptions } from './acp-session-runtime' @@ -42,9 +46,16 @@ export function createAcpAgentConnection( return new AcpAgentConnection(launch, options, spawnImpl) } +type AcpConnectionLifecycle = { + closing: boolean + error?: Error + finishExit?: () => void + outputEndTimer?: ReturnType +} + export class AcpAgentConnection extends AcpSessionRuntime { private readonly managed: ManagedProviderProcess - private readonly lifecycle: { closing: boolean; error?: Error } + private readonly lifecycle: AcpConnectionLifecycle private readonly diagnoseLifecycle: (message: string) => void readonly spawned: Promise @@ -86,7 +97,7 @@ export class AcpAgentConnection extends AcpSessionRuntime { } : {}) }) - const lifecycle: { closing: boolean; error?: Error } = { closing: false } + const lifecycle: AcpConnectionLifecycle = { closing: false } const diagnose = (message: string): void => { try { options.onDiagnostic?.(message) @@ -99,6 +110,12 @@ export class AcpAgentConnection extends AcpSessionRuntime { peer, onOtherLine: (line, cut) => signInLine(line, 'stdout', cut), onAuthAnswered: (signedIn) => signIn?.answered?.(signedIn), + // A closed stdin usually means the agent is exiting; its exit settles calls once its output + // drained, so the bound only covers an agent that stopped reading without exiting. + onOutputEnd: (_error, finish) => { + lifecycle.outputEndTimer = setTimeout(finish, PROVIDER_EXIT_OUTPUT_DRAIN_MS) + lifecycle.outputEndTimer.unref() + }, onClose: (error) => { lifecycle.error ??= error if (lifecycle.closing || managed.rootVerdict === 'exited') { @@ -142,16 +159,31 @@ export class AcpAgentConnection extends AcpSessionRuntime { super.close(new AcpConnectionClosedError('The agent printed a sign-in link Orca cannot read')) child.stderr.on('error', (error) => super.close(error)) managed.onExit((exit) => { - const error = withProviderSpawnAbsence( - lifecycle.error ?? - new AcpConnectionClosedError(managed.stderrTail().trim() || `${launch.command} exited`), - managed.spawnAbsence - ) - super.close(error) - try { - options.onExit?.(error, { expected: lifecycle.closing, exit }) - } catch (failure) { - diagnose(`ACP exit listener failed: ${String(failure)}`) + const expected = lifecycle.closing + const finish = (): void => { + clearTimeout(lifecycle.outputEndTimer) + const detail = managed.stderrTail().trim() || `${launch.command} exited` + // Only an exit Orca neither caused nor preceded with its own refusal leaves calls unanswered. + const error = withProviderSpawnAbsence( + lifecycle.error ?? + (expected + ? new AcpConnectionClosedError(detail) + : new AcpAgentExitedError(detail, managed.stderrTail().trim())), + managed.spawnAbsence + ) + super.close(error) + try { + options.onExit?.(error, { expected, exit }) + } catch (failure) { + diagnose(`ACP exit listener failed: ${String(failure)}`) + } + } + // A closed connection has no call left for the agent's output to answer. + if (lifecycle.closing || exit.processless || this.closed) { + finish() + } else { + // An answer written just before exit must still settle its call. + lifecycle.finishExit = afterProviderOutputDrains(child.stdout, finish, child) } }) } @@ -193,10 +225,11 @@ export class AcpAgentConnection extends AcpSessionRuntime { return this.managed.exitStatus } - onExit(listener: (exit: ProviderProcessExit) => void): void { + /** `requested`: Orca was closing the agent when it exited. */ + onExit(listener: (exit: ProviderProcessExit, requested?: boolean) => void): void { this.managed.onExit((exit) => { try { - listener(exit) + listener(exit, this.lifecycle.closing) } catch (failure) { this.diagnoseLifecycle(`ACP exit listener failed: ${String(failure)}`) } @@ -213,6 +246,8 @@ export class AcpAgentConnection extends AcpSessionRuntime { override close(error?: Error): Promise { this.lifecycle.closing ||= this.managed.rootVerdict !== 'exited' + clearTimeout(this.lifecycle.outputEndTimer) + this.lifecycle.finishExit?.() this.drainNotifications(error) return this.managed .close() diff --git a/src/main/acp/acp-errors.ts b/src/main/acp/acp-errors.ts index f8f6a480c3fe..2a863c746af9 100644 --- a/src/main/acp/acp-errors.ts +++ b/src/main/acp/acp-errors.ts @@ -1,5 +1,9 @@ import type { AuthMethod } from './generated/acp-protocol.generated' import { withProviderRequestTimeout } from '../provider-process/provider-request-timeout' +import { + providerDiagnostic, + withProviderDiagnostic +} from '../../shared/agent-session/failures/agent-session-failure' export class AcpRpcError extends Error { constructor( @@ -76,6 +80,18 @@ export class AcpConnectionClosedError extends Error { } } +/** The agent's process ended on its own and its output drained with the call still unanswered. + * Its stderr is the agent's own account, shown behind Details. */ +export class AcpAgentExitedError extends AcpConnectionClosedError { + constructor(message: string, stderr = '') { + super(message) + this.name = 'AcpAgentExitedError' + if (stderr) { + withProviderDiagnostic(this, providerDiagnostic(stderr, 'person')) + } + } +} + export class AcpRequestTimeoutError extends Error { constructor( readonly method: string, diff --git a/src/main/acp/acp-json-rpc-peer.test.ts b/src/main/acp/acp-json-rpc-peer.test.ts index 8d2a78bf0a51..79a419f2ee47 100644 --- a/src/main/acp/acp-json-rpc-peer.test.ts +++ b/src/main/acp/acp-json-rpc-peer.test.ts @@ -9,6 +9,7 @@ import { } from './acp-errors' import { AcpScriptedAgent, deferred, tick } from './acp-scripted-agent.test-support' import { providerRequestTimeoutOf } from '../provider-process/provider-request-timeout' +import captures from '../native-chat/agent-session-wire/__fixtures__/startup-protocol-rejections.json' const peers: AcpJsonRpcPeer[] = [] const agents: AcpScriptedAgent[] = [] @@ -382,4 +383,46 @@ describe('ACP JSON-RPC peer', () => { expect(peer.closed).toBe(true) input.destroy() }) + + it('settles a call with the final answer the agent wrote after it stopped reading', async () => { + const { response } = captures.acp + const { peer, agent } = fixture() + const asked = deferred() + agent.on(captures.acp.request.method, asked.resolve) + const load = peer.request(captures.acp.request.method, {}).catch((error: unknown) => error) + await asked.promise + // The agent closes its stdin before the last chunk of its answer reaches Orca. + agent.stdin.emit('finish') + await expect(peer.request('late', {})).rejects.toBeInstanceOf(AcpConnectionClosedError) + agent.stdout.write(`${JSON.stringify({ ...response, id: 1 })}\n`) + expect(await load).toMatchObject({ code: response.error.code, data: response.error.data }) + expect(peer.closed).toBe(true) + }) + + it('keeps an earlier call answerable when a later write fails', async () => { + const input = new PassThrough() + let writes = 0 + const output = new Writable({ + write(_chunk, _encoding, callback) { + writes += 1 + if (writes === 1) { + callback() + } else { + setImmediate(() => callback(new Error('Broken pipe from write'))) + } + } + }) + const peer = new AcpJsonRpcPeer(input, output) + peers.push(peer) + const first = peer.request('session/new', {}).catch((error: unknown) => error) + await tick() + const second = peer.request('session/set_mode', {}).catch((error: unknown) => error) + await tick() + input.write( + `${JSON.stringify({ jsonrpc: '2.0', id: 1, error: { code: -32600, message: 'refused' } })}\n` + ) + expect(await first).toBeInstanceOf(AcpRpcError) + input.end() + expect(await second).toMatchObject({ message: 'Broken pipe from write' }) + }) }) diff --git a/src/main/acp/acp-json-rpc-peer.ts b/src/main/acp/acp-json-rpc-peer.ts index 4e82096b5082..09defd97acd1 100644 --- a/src/main/acp/acp-json-rpc-peer.ts +++ b/src/main/acp/acp-json-rpc-peer.ts @@ -1,5 +1,4 @@ import type { Readable, Writable } from 'node:stream' -import { z } from 'zod' import { readAcpResponse } from './acp-response-settlement' import type { AcpResponse, AcpRequestOptions, AcpPendingResponse } from './acp-response-settlement' import { @@ -13,12 +12,9 @@ export type { AcpPeerOptions } from './acp-peer-limits' import { settleOversizedAcpLine } from './acp-oversized-lines' import { AcpWriteQueue } from './acp-write-queue' import { detachAcpStreamErrorHandler } from './acp-stdio-error-boundary' -import { - acpRpcEnvelopeSchema, - invalidAcpResponseEnvelope, - settleAcpResponse, - type AcpJsonRpcMessage -} from './acp-rpc-envelope' +import { ProviderOutputEnd } from '../provider-process/provider-output-end' +import { acpRpcEnvelopeSchema, type AcpJsonRpcMessage } from './acp-rpc-envelope' +import { rejectUnreadableAcpReply, settleAcpReply } from './acp-peer-reply' export type { AcpJsonRpcMessage } from './acp-rpc-envelope' export type AcpRequestContext = { id: string | number | null; signal: AbortSignal } export type AcpPeerHandlers = { @@ -27,6 +23,9 @@ export type AcpPeerHandlers = { onNotification?: (method: string, params: unknown) => void onDiagnostic?: (message: string) => void onClose?: (error: Error) => void + /** Bounds how long calls outstanding when the agent stopped reading wait for answers it wrote; + * `finish` settles them. Defaults to the drain of the agent's output. */ + onOutputEnd?: (error: Error, finish: () => void) => void /** Includes non-protocol output and unterminated final lines. */ onOtherLine?: (line: string, cut?: boolean) => void } @@ -37,19 +36,20 @@ export class AcpJsonRpcPeer { private readonly framer: ReturnType private nextId = 1 private terminalError?: Error + private readonly outputEnd: ProviderOutputEnd private drainingNotifications = false private readonly limits: Required constructor( private readonly input: Readable, - private readonly output: Writable, + output: Writable, private readonly handlers: AcpPeerHandlers = {}, options: AcpPeerOptions = {} ) { const limits = resolveAcpPeerOptions(options) this.limits = limits this.writer = new AcpWriteQueue(output, limits.maxQueuedWriteBytes, (error) => - this.close(error) + this.endOutput(error) ) this.incoming = new AcpIncomingRequests( handlers.onRequest, @@ -80,15 +80,15 @@ export class AcpJsonRpcPeer { input.on('end', this.onInputEnd) input.on('close', this.onInputEnd) input.on('error', this.onError) - output.on('close', this.onEnd) - output.on('finish', this.onEnd) - output.on('error', this.onError) - if ( - (this.limits.closeOnInputEnd && (input.destroyed || input.readableEnded)) || - output.destroyed || - !output.writable - ) { + this.outputEnd = new ProviderOutputEnd( + output, + (error) => this.endOutput(error), + () => new AcpConnectionClosedError() + ) + if (this.limits.closeOnInputEnd && (input.destroyed || input.readableEnded)) { this.onEnd() + } else if (output.destroyed || !output.writable) { + this.endOutput(new AcpConnectionClosedError()) } } @@ -107,8 +107,8 @@ export class AcpJsonRpcPeer { params: unknown, options: Partial = {} ): Promise { - if (this.terminalError) { - return Promise.reject(this.terminalError) + if (this.terminalError ?? this.outputEnd.error) { + return Promise.reject(this.terminalError ?? this.outputEnd.error) } if (this.pending.size >= this.limits.maxPendingRequests) { return Promise.reject(new Error('ACP pending request capacity exceeded')) @@ -148,7 +148,8 @@ export class AcpJsonRpcPeer { }) void this.send({ jsonrpc: '2.0', id, method, params }, controller.signal).catch((error) => { const pending = this.pending.get(id) - if (!pending) { + // A write lost to the agent no longer reading leaves the call to the output drain. + if (!pending || this.outputEnd.error) { return } this.pending.delete(id) @@ -191,9 +192,7 @@ export class AcpJsonRpcPeer { if (this.terminalError) { return } - this.output.removeListener('close', this.onEnd) - this.output.removeListener('finish', this.onEnd) - detachAcpStreamErrorHandler(this.output, this.onError) + this.outputEnd.detach() this.terminalError = error this.writer.close(error) for (const pending of this.pending.values()) { @@ -217,13 +216,26 @@ export class AcpJsonRpcPeer { } } private readonly onEnd = (): void => this.close() + + private endOutput(error: Error): void { + if (this.terminalError || !this.outputEnd.begin(error)) { + return + } + this.writer.close(error) + this.outputEnd.settle( + this.pending.size > 0, + this.input, + () => this.close(error), + this.handlers.onOutputEnd && ((finish) => this.handlers.onOutputEnd?.(error, finish)) + ) + } private readonly onInputEnd = (): void => { const tail = this.framer.takeTail() if (tail.trim()) { this.otherLine(tail) } if (this.limits.closeOnInputEnd) { - this.close() + this.close(this.outputEnd.error) } } private readonly onError = (error: Error): void => this.close(error) @@ -266,12 +278,7 @@ export class AcpJsonRpcPeer { const parsed = acpRpcEnvelopeSchema.safeParse(record) if (!parsed.success) { this.diagnose('Ignored invalid ACP JSON-RPC envelope') - const response = z - .object({ id: z.number(), method: z.undefined().optional() }) - .safeParse(record) - if (response.success) { - this.rejectPending(response.data.id, invalidAcpResponseEnvelope(record)) - } + rejectUnreadableAcpReply(record, (id, error) => this.rejectPending(id, error)) return } const frame = parsed.data @@ -294,22 +301,15 @@ export class AcpJsonRpcPeer { } return } - if ('result' in frame === 'error' in frame) { - this.diagnose('Ignored invalid ACP response') - if (typeof frame.id === 'number') { - this.rejectPending(frame.id, invalidAcpResponseEnvelope(record)) - } - return - } - if (typeof frame.id !== 'number') { - return - } - const pending = this.pending.get(frame.id) - if (!pending) { - return + settleAcpReply({ + record, + frame, + pending: this.pending, + rejectPending: (id, error) => this.rejectPending(id, error), + diagnose: (message) => this.diagnose(message) + }) + if (this.outputEnd.error && this.pending.size === 0) { + this.close(this.outputEnd.error) } - this.pending.delete(frame.id) - clearTimeout(pending.timer) - settleAcpResponse(frame, pending, (message) => this.diagnose(message)) } } diff --git a/src/main/acp/acp-peer-reply.ts b/src/main/acp/acp-peer-reply.ts new file mode 100644 index 000000000000..4366f8fd783b --- /dev/null +++ b/src/main/acp/acp-peer-reply.ts @@ -0,0 +1,42 @@ +import { z } from 'zod' +import type { AcpPendingResponse } from './acp-response-settlement' +import { + invalidAcpResponseEnvelope, + settleAcpResponse, + type AcpJsonRpcMessage +} from './acp-rpc-envelope' + +type RejectPending = (id: number, error: (method: string) => Error) => void + +/** A record no envelope reads still fails the call its id names. */ +export function rejectUnreadableAcpReply(record: unknown, rejectPending: RejectPending): void { + const response = z.object({ id: z.number(), method: z.undefined().optional() }).safeParse(record) + if (response.success) { + rejectPending(response.data.id, invalidAcpResponseEnvelope(record)) + } +} + +/** Settles the outstanding call an agent's reply answers; a malformed reply fails its call. */ +export function settleAcpReply(input: { + record: unknown + frame: AcpJsonRpcMessage + pending: Map + rejectPending: RejectPending + diagnose: (message: string) => void +}): void { + const { frame } = input + if ('result' in frame === 'error' in frame) { + input.diagnose('Ignored invalid ACP response') + if (typeof frame.id === 'number') { + input.rejectPending(frame.id, invalidAcpResponseEnvelope(input.record)) + } + return + } + const pending = typeof frame.id === 'number' ? input.pending.get(frame.id) : undefined + if (!pending || typeof frame.id !== 'number') { + return + } + input.pending.delete(frame.id) + clearTimeout(pending.timer) + settleAcpResponse(frame, pending, input.diagnose) +} diff --git a/src/main/acp/acp-scripted-connection.test-fixture.ts b/src/main/acp/acp-scripted-connection.test-fixture.ts index 05f15498a1fd..c121312c3348 100644 --- a/src/main/acp/acp-scripted-connection.test-fixture.ts +++ b/src/main/acp/acp-scripted-connection.test-fixture.ts @@ -5,7 +5,7 @@ import { type ProviderSpawnAbsence } from '../provider-process/provider-executable-missing' import type { AcpAgentConnectionOptions } from './acp-agent-connection' -import { AcpConnectionClosedError } from './acp-errors' +import { AcpAgentExitedError, AcpConnectionClosedError } from './acp-errors' import { AcpSessionRuntime } from './acp-session-runtime' import type { AcpStructuredConnection } from './acp-structured-connection' import { AcpScriptedAgent } from './acp-scripted-agent.test-support' @@ -80,7 +80,10 @@ export class FakeAcpChild extends AcpSessionRuntime implements AcpStructuredConn this.exitedWith = status // As the real connection: a spawn ENOENT marks the exit error with what it proved absent. const error = withProviderSpawnAbsence( - this.lostWith ?? new AcpConnectionClosedError(this.stderr || `${this.launch.command} exited`), + this.lostWith ?? + (this.closing + ? new AcpConnectionClosedError(this.stderr || `${this.launch.command} exited`) + : new AcpAgentExitedError(this.stderr || `${this.launch.command} exited`, this.stderr)), this.spawnAbsence ) super.close(error) diff --git a/src/main/acp/acp-session-reopen-failure.test.ts b/src/main/acp/acp-session-reopen-failure.test.ts index 84ce618fa9ef..16a19955047d 100644 --- a/src/main/acp/acp-session-reopen-failure.test.ts +++ b/src/main/acp/acp-session-reopen-failure.test.ts @@ -43,6 +43,8 @@ it('propagates a temporary load envelope before takeover or a dead-child asserti ) }) await rig.acquire() + // The adapter answers for its own declared-temporary answer, which it keeps as the cause. + expect(rig.openVerdict()).toMatchObject({ retry: 'safe' }) expect(rig.openFailure()).toBeInstanceOf(AcpAgentError) expect(rig.openFailure()).toMatchObject({ data: { errorName: 'ServerUnavailable' } }) expect(rig.sent('session/new')).toEqual([]) diff --git a/src/main/acp/acp-session-reopen-failure.ts b/src/main/acp/acp-session-reopen-failure.ts index 3da135aa3f01..bc515b2b1f49 100644 --- a/src/main/acp/acp-session-reopen-failure.ts +++ b/src/main/acp/acp-session-reopen-failure.ts @@ -5,7 +5,7 @@ import type { ProviderTimelineEvent } from '../native-chat/agent-session-timelin import { AcpAuthRequiredError, AcpRpcError } from './acp-errors' import type { AcpStructuredLaunch } from './acp-structured-launch-resolution' -import { temporaryAcpStartupAnswer } from './acp-startup-failure' +import { temporaryAcpStartupAnswer } from './acp-startup-outcome' const ACP_RESOURCE_NOT_FOUND = -32002 export const ACP_REOPEN_FAILED = 'ACP agent could not reopen its saved session; starting a new one' diff --git a/src/main/acp/acp-session-runtime.ts b/src/main/acp/acp-session-runtime.ts index dc2b87e9bb51..1d2a14e1bb9f 100644 --- a/src/main/acp/acp-session-runtime.ts +++ b/src/main/acp/acp-session-runtime.ts @@ -2,7 +2,12 @@ import type { Readable, Writable } from 'node:stream' import { z } from 'zod' import { AcpRpcError } from './acp-errors' import { callAcpSession } from './acp-response-settlement' -import { AcpJsonRpcPeer, type AcpPeerOptions, type AcpRequestContext } from './acp-json-rpc-peer' +import { + AcpJsonRpcPeer, + type AcpPeerHandlers, + type AcpPeerOptions, + type AcpRequestContext +} from './acp-json-rpc-peer' import { answerAcpPermission, readAcpPermissionRequest, @@ -51,6 +56,7 @@ export type AcpSessionRuntimeOptions = { onExtensionNotification?: (method: string, params: unknown) => void onDiagnostic?: (message: string) => void onClose?: (error: Error) => void + onOutputEnd?: AcpPeerHandlers['onOutputEnd'] onOtherLine?: (line: string, cut?: boolean) => void /** Authentication answers, published synchronously as responses are accepted. */ onAuthAnswered?: (signedIn: boolean) => void @@ -79,6 +85,7 @@ export class AcpSessionRuntime { onNotification: (method, params) => this.handleNotification(method, params), onDiagnostic: options.onDiagnostic, onClose: options.onClose, + ...(options.onOutputEnd ? { onOutputEnd: options.onOutputEnd } : {}), onOtherLine: options.onOtherLine }, options.peer diff --git a/src/main/acp/acp-startup-failure.ts b/src/main/acp/acp-startup-failure.ts deleted file mode 100644 index 1b1d71a03910..000000000000 --- a/src/main/acp/acp-startup-failure.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { AcpAgentError } from './acp-errors' - -export function temporaryAcpStartupAnswer(error: unknown): boolean { - return ( - error instanceof AcpAgentError && - error.code === -32603 && - typeof error.data === 'object' && - error.data !== null && - 'errorName' in error.data && - (error.data.errorName === 'ServerUnavailable' || error.data.errorName === 'CatalogNotReady') - ) -} diff --git a/src/main/acp/acp-startup-outcome.ts b/src/main/acp/acp-startup-outcome.ts new file mode 100644 index 000000000000..aa6e1118b5f4 --- /dev/null +++ b/src/main/acp/acp-startup-outcome.ts @@ -0,0 +1,44 @@ +import { + agentSessionAcquireFailure, + agentSessionStartFailure, + type AgentSessionStartAnswer, + type AgentSessionStartFailure +} from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { + AcpAgentError, + AcpAgentExitedError, + AcpFrameTooLargeError, + AcpRpcError +} from './acp-errors' + +export function temporaryAcpStartupAnswer(error: unknown): boolean { + return ( + error instanceof AcpAgentError && + error.code === -32603 && + typeof error.data === 'object' && + error.data !== null && + 'errorName' in error.data && + (error.data.errorName === 'ServerUnavailable' || error.data.errorName === 'CatalogNotReady') + ) +} + +/** An agent's answer, sign-in included, is final unless it declared a temporary one; a frame + * Orca could not read is final too. Only the process's own drained exit is unanswered: a closed + * connection alone may be Orca refusing what the agent wrote. */ +export const acpStartupAnswer: AgentSessionStartAnswer = (cause) => { + if (cause instanceof AcpRpcError) { + return temporaryAcpStartupAnswer(cause) ? 'temporary' : 'final' + } + if (cause instanceof AcpFrameTooLargeError) { + return 'final' + } + return cause instanceof AcpAgentExitedError ? 'exited' : undefined +} + +export function acpStartFailure(error: unknown, exited = false): AgentSessionStartFailure { + return agentSessionStartFailure(error, acpStartupAnswer, exited) +} + +export function acpAcquireFailure(error: unknown): unknown { + return agentSessionAcquireFailure(error, acpStartupAnswer) +} diff --git a/src/main/acp/acp-structured-adapter.test-support.ts b/src/main/acp/acp-structured-adapter.test-support.ts index 2f4691585981..96d28fd26a30 100644 --- a/src/main/acp/acp-structured-adapter.test-support.ts +++ b/src/main/acp/acp-structured-adapter.test-support.ts @@ -19,6 +19,7 @@ import { type AcpScriptedAgent, tick, type FakeFrame } from './acp-scripted-agen import type { AcpStructuredLaunch } from './acp-structured-launch-resolution' import { AcpStructuredSessionAdapter } from './acp-structured-session-adapter' import type { AcpStructuredSessionAdapterDeps } from './acp-structured-session-adapter-deps' +import { AgentSessionStartFailure } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' export const GROK = acpLaunchSpecFor('grok')! const RIG_ENV = { PATH: '/usr/bin', ORCA_PANE_KEY: 'tab-1:pane-1', ORCA_AGENT_HOOK_PORT: '1234' } @@ -57,7 +58,10 @@ export type AcpAdapterRig = { lifecycle: StructuredAgentSessionLifecycleEvent[] readonly ended: Extract[] started(): StructuredAgentSessionProviderOpenResult + /** The error the adapter's verdict carries, as the agent or connection gave it. */ openFailure(): unknown + /** The adapter's verdict on the failed open. */ + openVerdict(): AgentSessionStartFailure | undefined settled: Parameters>[0][] acquire(options?: { fence?: number @@ -86,7 +90,7 @@ export async function openAcpAdapterRig( const spawned: string[] = [] const lifecycle: StructuredAgentSessionLifecycleEvent[] = [] const settled: AcpAdapterRig['settled'] = [] - let openFailure: unknown + let openVerdict: AgentSessionStartFailure | undefined let opened: StructuredAgentSessionProviderOpenResult | null = null const startListeners = new Set<() => void>() const spec = options.spec ?? GROK @@ -185,10 +189,11 @@ export async function openAcpAdapterRig( } return opened }, - openFailure: () => openFailure, + openFailure: () => openVerdict?.cause, + openVerdict: () => openVerdict, settled, acquire: async (acquireOptions = {}) => { - openFailure = undefined + openVerdict = undefined opened = null const acquisition = await adapter.acquire({ identity: { @@ -218,7 +223,10 @@ export async function openAcpAdapterRig( signal: acquireOptions.signal ?? new AbortController().signal }) } catch (error) { - openFailure = error + if (!(error instanceof AgentSessionStartFailure)) { + throw new Error('the ACP adapter must answer a failed open with its verdict') + } + openVerdict = error await adapter.closeSession(SESSION) } } diff --git a/src/main/acp/acp-structured-child-lifecycle.ts b/src/main/acp/acp-structured-child-lifecycle.ts index 2aaa3bf4854d..4459c05b794e 100644 --- a/src/main/acp/acp-structured-child-lifecycle.ts +++ b/src/main/acp/acp-structured-child-lifecycle.ts @@ -22,6 +22,9 @@ import { neverThrowingStructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' import { windDownAcpTurn } from './acp-structured-stop' +import { AcpAgentExitedError } from './acp-errors' +import { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { acpStartFailure } from './acp-startup-outcome' /** An agent that never ran names its missing CLI or folder; any other end while starting is its exit. */ function acpStartupEndFailure(session: AcpStructuredChild): SubmissionRejectionFact { @@ -37,6 +40,8 @@ function acpStartupEndFailure(session: AcpStructuredChild): SubmissionRejectionF export class AcpStructuredChildren { readonly sessions = new Map() + /** Children that ended before they were ready answer for that ending, open included. */ + readonly endings = new AgentSessionStartEndings() readonly starts = new ProviderAcquisitionStarts() constructor(private readonly deps: AcpStructuredSessionAdapterDeps) {} capturePendingOutput: StructuredAgentSessionAdapter['capturePendingOutput'] = (input) => { @@ -184,6 +189,21 @@ export class AcpStructuredChildren { } } + /** Taken when the process exit is first seen, so a start settling meanwhile cannot miss it. */ + recordExit(sessionId: string, generation: string, connection: AcpStructuredConnection): void { + const detail = + connection.stderrTail() || `${this.deps.spec.agent} ACP agent ended while starting` + const exited = withProviderSpawnAbsence( + new AcpAgentExitedError(detail, connection.stderrTail()), + connection.spawnAbsence + ) + this.endings.record(sessionId, generation, acpStartFailure(exited)) + } + + private recordEnding(session: AcpStructuredChild, error: Error): void { + this.endings.record(session.sessionId, session.acquisitionGeneration, acpStartFailure(error)) + } + /** Close the journal on protocol loss; notify the host after process exit is proven. */ connectionLost(session: AcpStructuredChild | null, error: Error): void { if ( @@ -193,6 +213,8 @@ export class AcpStructuredChildren { ) { return } + // Orca refusing what the agent wrote outranks the exit its cleanup then causes. + this.recordEnding(session, error) if (!isAcpReadySession(session)) { session.journalClosed = error.message || error.name return diff --git a/src/main/acp/acp-structured-session-adapter.ts b/src/main/acp/acp-structured-session-adapter.ts index 967567945159..3cad7b0f5a75 100644 --- a/src/main/acp/acp-structured-session-adapter.ts +++ b/src/main/acp/acp-structured-session-adapter.ts @@ -1,11 +1,9 @@ import { randomUUID } from 'node:crypto' -import { withProviderDiagnostic } from '../../shared/agent-session/failures/agent-session-failure' import type { AgentSessionExecutionLocation } from '../../shared/agent-session/lifecycle/agent-session-record' import { AgentSessionAcquisitionExitProvenError, AgentSessionAcquisitionExitUnprovenError, AgentSessionAcquisitionRootExitObservedError, - AgentSessionAcquisitionRefusal, AgentSessionPreSpawnError, isAgentSessionPreSpawnError, type AgentSessionAcquisition, @@ -39,6 +37,8 @@ import { AcpStructuredChildren } from './acp-structured-child-lifecycle' import { AcpConversationCopies } from './acp-conversation-copies' import { AcpTextTasks } from './acp-text-task' import { supportsAcpStructuredLocation } from './acp-structured-location-support' +import { acpAcquireFailure } from './acp-startup-outcome' +import { openAcpStart } from './acp-structured-start-open' export class AcpStructuredSessionAdapter implements SessionAdapter { readonly atRestCommands: AcpStructuredSessionAdapterDeps['atRestCommands'] @@ -65,6 +65,7 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { async acquire(input: StructuredAgentSessionAcquireInput): Promise { const sessionId = input.identity.sessionId + this.children.endings.forget(sessionId) const gate = this.deps.spec.launchGate?.enter(input.signal) const attempt = this.children.starts.begin(gate?.signal ?? input.signal) try { @@ -76,6 +77,8 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { ) } return await this.start({ ...input, signal: attempt.signal }, attempt) + } catch (error) { + throw acpAcquireFailure(error) } finally { this.children.starts.end(attempt) gate?.leave() @@ -95,6 +98,12 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { generation, abandoned: () => attempt.signal.aborted, track: (connection) => { + connection.onExit((_exit, requested) => { + if (!requested) { + input.events?.observeRootExit?.() + this.children.recordExit(sessionId, generation, connection) + } + }) this.children.starts.track(attempt, connection) this.deps.spec.launchGate?.retain(connection) }, @@ -151,30 +160,10 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { } } - open: NonNullable = async (input) => { - const child = this.children.sessions.get(input.sessionId) - if ( - !child || - child.fence !== input.fence || - child.acquisitionGeneration !== input.acquisitionGeneration || - child.ended - ) { - throw new Error('ACP child changed before it opened') - } - try { - return await child.open(input.signal) - } catch (error) { - const failure = childState.acpStartupFailure(child, error) - if (failure) { - const refusal = new AgentSessionAcquisitionRefusal( - `${acpAgentName(child.spec.agent)} requires sign-in`, - 'notSignedIn' - ) - throw failure.detail ? withProviderDiagnostic(refusal, failure.detail) : refusal - } - throw error - } - } + open: NonNullable = (input) => openAcpStart(this.children, input) + + startupEnding: NonNullable = (sessionId, generation) => + this.children.endings.get(sessionId, generation) async dispatch(input: Parameters[0]): Promise { const lost = this.children.sessions.get(input.sessionId) @@ -318,6 +307,7 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { forceCloseSession = (sessionId: string): Promise => this.children.close(sessionId, false) async closeAll(): Promise { + this.children.endings.clear() await this.children.closeAll([ this.atRestCommands?.closeAll?.(), this.copies.closeAll(), diff --git a/src/main/acp/acp-structured-start-open.ts b/src/main/acp/acp-structured-start-open.ts new file mode 100644 index 000000000000..cb9034eed3b8 --- /dev/null +++ b/src/main/acp/acp-structured-start-open.ts @@ -0,0 +1,45 @@ +import { withProviderDiagnostic } from '../../shared/agent-session/failures/agent-session-failure' +import { + AgentSessionAcquisitionRefusal, + type StructuredAgentSessionAdapter +} from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { StructuredAgentSessionProviderOpenInput } from '../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' +import { acpAgentName } from './acp-structured-agent-definitions' +import { acpStartupFailure } from './acp-structured-child' +import type { AcpStructuredChildren } from './acp-structured-child-lifecycle' +import { acpStartFailure } from './acp-startup-outcome' + +/** A start's open answers with the agent's verdict; a child that already ended answers with how + * it ended, and an answer its own exit followed is still worded as that exit. */ +export async function openAcpStart( + children: AcpStructuredChildren, + input: StructuredAgentSessionProviderOpenInput +): ReturnType> { + const child = children.sessions.get(input.sessionId) + try { + if ( + !child || + child.fence !== input.fence || + child.acquisitionGeneration !== input.acquisitionGeneration || + child.ended + ) { + throw ( + children.endings.get(input.sessionId, input.acquisitionGeneration) ?? + new Error('ACP child changed before it opened') + ) + } + return await child.open(input.signal).catch((error: unknown) => { + const failure = acpStartupFailure(child, error) + if (!failure) { + throw error + } + const refusal = new AgentSessionAcquisitionRefusal( + `${acpAgentName(child.spec.agent)} requires sign-in`, + 'notSignedIn' + ) + throw failure.detail ? withProviderDiagnostic(refusal, failure.detail) : refusal + }) + } catch (error) { + throw acpStartFailure(error, child?.connection.exited === true && !child.closeRequested) + } +} diff --git a/src/main/claude/claude-startup-failure.ts b/src/main/claude/claude-startup-failure.ts index e425ed63ca90..0727cc9f46bd 100644 --- a/src/main/claude/claude-startup-failure.ts +++ b/src/main/claude/claude-startup-failure.ts @@ -1,5 +1,12 @@ import { findInErrorChain } from '../provider-process/provider-error-chain' import { ClaudeControlRequestError } from './claude-agent-sdk-control-requests' +import { + agentSessionAcquireFailure, + agentSessionStartFailure, + providerResourcePressure, + type AgentSessionStartAnswer, + type AgentSessionStartFailure +} from '../native-chat/agent-session-wire/structured-agent-session-start-failure' /** SDK runtime tags are not in its public Error declaration. */ export function claudeStartupErrorClass(error: Error): string | undefined { @@ -8,39 +15,30 @@ export function claudeStartupErrorClass(error: Error): string | undefined { : undefined } -export function claudeStartupAnswer(error: Error): 'temporary' | 'final' | undefined { - const tag = claudeStartupErrorClass(error) +/** The SDK's tag says whether Claude's process ended unanswered; every other tag is its answer. + * A launch the host could not run for want of resources is left to the shared rule. */ +export const claudeStartupAnswer: AgentSessionStartAnswer = (error) => { + const tag = + claudeStartupErrorClass(error) ?? + (error instanceof ClaudeControlRequestError + ? (findInErrorChain(error.cause, (cause) => claudeStartupErrorClass(cause)) ?? 'answered') + : undefined) + if (tag === undefined) { + return undefined + } if (tag === 'process_exited_nonzero' || tag === 'process_killed_by_signal') { - return 'temporary' + return 'exited' } if (tag === 'executable_launch_failed' || tag === 'spawn_failed' || tag === 'process_error') { - return resourcePressure(error) ? 'temporary' : 'final' - } - if (tag !== undefined) { - return 'final' - } - if (error instanceof ClaudeControlRequestError) { - const origin = findInErrorChain(error.cause, (cause) => claudeStartupErrorClass(cause)) - if ( - origin === 'executable_launch_failed' || - origin === 'spawn_failed' || - origin === 'process_error' - ) { - return resourcePressure(error) ? 'temporary' : 'final' - } - return origin === 'process_exited_nonzero' || origin === 'process_killed_by_signal' - ? 'temporary' - : 'final' + return providerResourcePressure(error) ? undefined : 'final' } - return undefined + return 'final' +} + +export function claudeStartFailure(error: unknown): AgentSessionStartFailure { + return agentSessionStartFailure(error, claudeStartupAnswer) } -function resourcePressure(error: Error): boolean { - return ( - findInErrorChain(error, (cause) => - 'code' in cause && ['EAGAIN', 'EMFILE', 'ENFILE'].some((code) => cause.code === code) - ? true - : undefined - ) === true - ) +export function claudeAcquireFailure(error: unknown): unknown { + return agentSessionAcquireFailure(error, claudeStartupAnswer) } diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index 737a55c6a78c..7d9ab1adef06 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -26,12 +26,15 @@ import { claudeProviderHandle } from '../../shared/agent-session/lifecycle/agent import { providerExecutableMissing } from '../provider-process/provider-executable-missing' import { findInErrorChain } from '../provider-process/provider-error-chain' import { providerExitObserved } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' -import { retryableStructuredAgentSessionStart } from '../native-chat/agent-session-wire/structured-agent-session-start-retry' import { adapterAtPublishFor, fakeClaude, identityFor } from './structured/claude-structured-session-test-support' +import { claudeStartFailure } from './claude-startup-failure' + +const retryableStructuredAgentSessionStart = (error: unknown): boolean => + claudeStartFailure(error).retry === 'safe' // Exercise the real SDK against a scripted child CLI. const FAKE_CLI = join(__dirname, '__fixtures__', 'claude-agent-sdk-scripted-cli.mjs') diff --git a/src/main/claude/claude-stream-json-exit-error.test.ts b/src/main/claude/claude-stream-json-exit-error.test.ts index 0b1ccf418dfb..a7c08e707bd1 100644 --- a/src/main/claude/claude-stream-json-exit-error.test.ts +++ b/src/main/claude/claude-stream-json-exit-error.test.ts @@ -5,7 +5,10 @@ import { claudeStreamJsonExitError } from './claude-stream-json-exit-error' import { providerStartupFailureFact } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' -import { retryableStructuredAgentSessionStart } from '../native-chat/agent-session-wire/structured-agent-session-start-retry' +import { claudeStartFailure } from './claude-startup-failure' + +const retryableStructuredAgentSessionStart = (error: unknown): boolean => + claudeStartFailure(error).retry === 'safe' const frame = JSON.parse( readFileSync(new URL('./__fixtures__/claude-missing-conversation.jsonl', import.meta.url), 'utf8') diff --git a/src/main/claude/structured/claude-structured-session-acquisition.ts b/src/main/claude/structured/claude-structured-session-acquisition.ts index 160615c3f2c5..e782573d07b9 100644 --- a/src/main/claude/structured/claude-structured-session-acquisition.ts +++ b/src/main/claude/structured/claude-structured-session-acquisition.ts @@ -222,7 +222,7 @@ export async function acquireClaudeSession({ callbacks.finishClose(sessionId, attempt) return } - childFailed(...claudeStartupExit(error, attempt.connection)) + childFailed(...claudeStartupExit(error, attempt.connection, input.events)) callbacks.handleExit(sessionId, attempt, error) }) } diff --git a/src/main/claude/structured/claude-structured-session-adapter.ts b/src/main/claude/structured/claude-structured-session-adapter.ts index 392f4e6f5d9e..4d21b87a2989 100644 --- a/src/main/claude/structured/claude-structured-session-adapter.ts +++ b/src/main/claude/structured/claude-structured-session-adapter.ts @@ -1,6 +1,8 @@ -import { waitForPromiseWithSignal } from '../../../shared/abort-signal-reason' import { dispatchClaudeCommand } from './claude-structured-command-dispatch' import { AgentSessionAcquisitionRootExitObservedError } from '../../native-chat/agent-session-wire/structured-agent-session-acquisition-errors' +import { claudeAcquireFailure } from '../claude-startup-failure' +import { openClaudeStart } from './claude-structured-start-ending' +import { AgentSessionStartEndings } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' import type { StructuredAgentSessionAcquireInput, StructuredAgentSessionAdapter @@ -43,7 +45,6 @@ import { } from './journal/claude-structured-prompt-ownership' import { claudePromptCancelRoute } from './journal/claude-structured-prompt-replies' import { claudeForkForRewind } from '../claude-rewind-fork' -import { openClaudeStructuredSession } from './claude-structured-session-open' import { ClaudeConversationNaming } from '../claude-conversation-naming' export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAdapter { @@ -51,6 +52,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda private readonly acquisitions = new ClaudeAcquisitionRegistry() private readonly exits = new Map() private readonly settledExitErrors = new Map() + private readonly endings = new AgentSessionStartEndings() private readonly exitLifecycle: ClaudeExitLifecycle private readonly naming: ClaudeConversationNaming | undefined @@ -62,6 +64,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda sessions: this.sessions, exits: this.exits, settledExitErrors: this.settledExitErrors, + endings: this.endings, deps, emit: (session, event) => this.emit(session, event) } @@ -71,6 +74,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda acquire = (input: StructuredAgentSessionAcquireInput) => { this.settledExitErrors.delete(input.identity.sessionId) + this.endings.forget(input.identity.sessionId) const record = this.deps.namingRecord?.(input.identity.sessionId) const naming = this.naming?.acquire(input, record) return acquireClaudeSession({ @@ -98,7 +102,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } }).catch((error: unknown) => { naming?.failed() - throw error + throw claudeAcquireFailure(error) }) } @@ -219,27 +223,15 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda input, this.deps.requestTimeoutMs ) - open: NonNullable = async (request) => { - try { - return await openClaudeStructuredSession({ - request, - session: this.session(request.sessionId), - sessions: this.sessions, - deps: this.deps, - emit: (session, event) => this.emit(session, event) - }) - } catch (error) { - const exit = this.exits.get(request.sessionId) - // Publish an already-observed exit before the host treats this rejection as its own fault. - if ( - exit?.session.acquisitionGeneration === request.acquisitionGeneration && - exit.publication - ) { - await waitForPromiseWithSignal(exit.publication, request.signal) - } - throw error - } - } + open: NonNullable = (request) => + openClaudeStart({ + request, + sessions: this.sessions, + exits: this.exits, + endings: this.endings, + deps: this.deps, + emit: (session, event) => this.emit(session, event) + }) readOptions = (input: { sessionId: string; fence: number }) => readClaudeStructuredSessionOptions(this.session(input.sessionId)) releaseAcquisition = (input: { sessionId: string }): Promise => @@ -247,6 +239,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda acknowledgeSessionRelease = (sessionId: string): void => { this.settledExitErrors.delete(sessionId) } + startupEnding = (sessionId: string, generation: string) => this.endings.get(sessionId, generation) hasOwnedAcquisition = (sessionId: string): boolean => this.acquisitions.get(sessionId) !== undefined || this.sessions.has(sessionId) || @@ -296,14 +289,16 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } }) - closeAll = (): Promise => - closeAllClaudeSessions({ + closeAll = (): Promise => { + this.endings.clear() + return closeAllClaudeSessions({ sessions: this.sessions, acquisitions: this.acquisitions, exits: this.exits, closeSession: this.closeSession, closeExit: (sessionId) => this.releaseAcquisition({ sessionId }) }) + } private session(sessionId: string): ClaudeSession { const session = this.sessions.get(sessionId) diff --git a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts index 2adea166e165..77238390ba2e 100644 --- a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts @@ -14,6 +14,8 @@ import { settledClaudeTurnEndLeaf } from './claude-structured-resume-point' import { claudeHeldSends } from '../claude-command-lifecycle' +import type { AgentSessionStartEndings } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { claudeExitEnding } from './claude-structured-start-ending' import { claudeCloseReport, settleClaudeExitedSession } from './claude-structured-session-close' import { AgentSessionAcquisitionExitUnprovenError, @@ -32,6 +34,8 @@ export type ClaudeExitLifecycle = { exits: Map /** Preserve diagnostics for sends admitted before the host learns of the exit. */ settledExitErrors: Map + /** Taken when the exit is first seen, so a start settling meanwhile cannot miss it. */ + endings: AgentSessionStartEndings deps: Pick emit: (session: ClaudeSession, event: ClaudeStructuredSessionEvent) => void } @@ -48,6 +52,7 @@ export function observeClaudeSessionExit( } const observedAt = lifecycle.deps.now?.() ?? Date.now() session.events?.observeExit?.(claudeHeldSends(session), observedAt) + lifecycle.endings.record(sessionId, session.acquisitionGeneration, claudeExitEnding(error)) lifecycle.sessions.delete(sessionId) // Run tree cleanup before recovery; an exit callback proves only root exit. const cleanup = session.connection.close().catch(() => false) diff --git a/src/main/claude/structured/claude-structured-session-startup-exit.ts b/src/main/claude/structured/claude-structured-session-startup-exit.ts index a81fbeadc719..2d2f187c5e96 100644 --- a/src/main/claude/structured/claude-structured-session-startup-exit.ts +++ b/src/main/claude/structured/claude-structured-session-startup-exit.ts @@ -6,6 +6,7 @@ import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import type { ClaudeStreamJsonConnection } from '../claude-stream-json-connection' +import type { StructuredAgentSessionEventSink } from '../../native-chat/agent-session-wire/structured-agent-session-event-sink' /** A physical exit waits briefly for initialize's answer; a delivered answer outranks the exit. */ export async function claudeStartupAnswerAfterExit( @@ -38,11 +39,14 @@ export async function claudeStartupError( : (connection.startupFailure?.() ?? error) } -/** An unexpected exit, and when it ends the start: a delivered initialize answer outranks it. */ +/** An unexpected exit, and when it ends the start: a delivered initialize answer outranks it. The + * start's root-exit receipt is taken now, before the answer join can delay it. */ export function claudeStartupExit( error: Error, - connection: ClaudeStreamJsonConnection | null + connection: ClaudeStreamJsonConnection | null, + events?: Pick ): [exited: Error, ended: Promise] { + events?.observeRootExit?.() // Mark in place because the fault report may retain this error object. const exited = withObservedProviderExit(error) return [exited, claudeStartupError(exited, connection)] diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index d86a55326c45..9f06ba2f6832 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -1,11 +1,8 @@ import sdkExit from '../fixtures/startup-sdk-process-exit.json' import { afterEach, expect, it, vi } from 'vitest' +import { findInErrorChain } from '../../provider-process/provider-error-chain' import { claudeStartupAnswerObservation } from '../claude-startup-answer-observation' import { runClaudeControl } from '../claude-agent-sdk-control-requests' -import { - retryableStructuredAgentSessionAcquisition, - retryableStructuredAgentSessionStart -} from '../../native-chat/agent-session-wire/structured-agent-session-start-retry' import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import { PROVIDER_SESSION_ID, @@ -19,6 +16,13 @@ import { import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' import { ClaudeStructuredSessionAdapter } from './claude-structured-session-adapter' import { NDJSON_MAX_LINE_BYTES } from '../../../shared/main-process-ndjson-framer' +import { claudeAcquireFailure, claudeStartFailure } from '../claude-startup-failure' +import { retryableAgentSessionAcquisition } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' + +const retryableStructuredAgentSessionStart = (error: unknown): boolean => + claudeStartFailure(error).retry === 'safe' +const retryableStructuredAgentSessionAcquisition = (error: unknown): boolean => + retryableAgentSessionAcquisition(claudeAcquireFailure(error)) afterEach(() => vi.useRealTimers()) @@ -64,7 +68,7 @@ it.each(['control_request_failed', 'process_exited_nonzero'])( ) await vi.advanceTimersByTimeAsync(2_000) const error = await result - expect(error.cause).toMatchObject({ errorClass, exitCode: 1 }) + expect(error.cause.cause).toMatchObject({ errorClass, exitCode: 1 }) expect(retryableStructuredAgentSessionStart(error)).toBe( errorClass === 'process_exited_nonzero' ) @@ -106,7 +110,7 @@ it('retains a failed result before a stalled SDK transcript flush and never wait connection.handlers.onExit?.(withObservedProviderExit(new Error('exited')), { expected: false }) await vi.advanceTimersByTimeAsync(2_100) const error = await result - expect(error.errorClass).toBe('error_result') + expect(error.cause.errorClass).toBe('error_result') expect(retryableStructuredAgentSessionStart(error)).toBe(false) expect(persist).not.toHaveBeenCalled() expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) @@ -174,7 +178,7 @@ it('settles an unanswered Claude startup exit with the same bounded recovery eli connection.exitVerdict = { root: 'exited', tree: 'exited' } connection.handlers.onExit?.(error, { expected: false }) await vi.advanceTimersByTimeAsync(2_100) - expect(await result).toBe(error) + expect((await result).cause).toBe(error) expect(retryableStructuredAgentSessionStart(await result)).toBe(true) claude.dispose() await adapter.closeAll() @@ -225,7 +229,9 @@ it('keeps a rejected initialize final when the child exits before acquisition pu }) startTime.resolve(1_700_000_000_000) const error = await result - expect(error.cause.cause).toMatchObject({ errorClass: 'control_request_failed' }) + expect( + findInErrorChain(error, (cause) => ('errorClass' in cause ? cause.errorClass : undefined)) + ).toBe('control_request_failed') expect(retryableStructuredAgentSessionAcquisition(error)).toBe(false) claude.dispose() connection.close = async () => true diff --git a/src/main/claude/structured/claude-structured-session-startup.test.ts b/src/main/claude/structured/claude-structured-session-startup.test.ts index 67c7a9e7569a..7ab47856e48f 100644 --- a/src/main/claude/structured/claude-structured-session-startup.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup.test.ts @@ -288,7 +288,7 @@ describe('Claude host-awaited open', () => { connection.handlers.onFault?.(fault) claudeFrame(connection, { ...MISSING, session_id: PROVIDER_SESSION_ID }) release.resolve({ ok: true }) - expect(await open).toBe(fault) + expect(await open).toMatchObject({ cause: fault }) expect(errorRows(appendItem)).toHaveLength(rows) } ) diff --git a/src/main/claude/structured/claude-structured-start-ending.ts b/src/main/claude/structured/claude-structured-start-ending.ts new file mode 100644 index 000000000000..99be5dc066b5 --- /dev/null +++ b/src/main/claude/structured/claude-structured-start-ending.ts @@ -0,0 +1,68 @@ +import { waitForPromiseWithSignal } from '../../../shared/abort-signal-reason' +import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import type { + AgentSessionStartEndings, + AgentSessionStartFailure +} from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' +import type { + StructuredAgentSessionProviderOpenInput, + StructuredAgentSessionProviderOpenResult +} from '../../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' +import { openClaudeStructuredSession } from './claude-structured-session-open' +import { claudeStartFailure } from '../claude-startup-failure' +import type { ClaudeSession, ClaudeSessionExit } from './claude-structured-session-state' + +/** Claude ended on its own: unanswered, unless what ended it was its answer or a refusal. */ +export function claudeExitEnding(error: Error): AgentSessionStartFailure { + return claudeStartFailure(withObservedProviderExit(new Error(error.message, { cause: error }))) +} + +/** A start's open answers with Claude's verdict. An exit already seen is published first, so the + * host never reads the rejection before the exit that explains it. */ +export async function openClaudeStart( + input: Omit[0], 'session'> & { + exits: Map + endings: AgentSessionStartEndings + } +): Promise { + const { request } = input + const session = input.sessions.get(request.sessionId) + if (!session) { + throw await claudeEndedStart( + input.exits.get(request.sessionId)?.session, + request, + input.endings + ) + } + try { + return await openClaudeStructuredSession({ ...input, session }) + } catch (error) { + const exit = input.exits.get(request.sessionId) + if (exit?.session.acquisitionGeneration === request.acquisitionGeneration && exit.publication) { + await waitForPromiseWithSignal(exit.publication, request.signal) + } + throw claudeStartFailure(error) + } +} + +/** A start that ended before its open answers with its own ending: the cause its startup settled + * on once the exit's initialize join finished, else the ending taken at its exit. */ +async function claudeEndedStart( + exited: ClaudeSession | undefined, + request: StructuredAgentSessionProviderOpenInput, + endings: AgentSessionStartEndings +): Promise { + const { sessionId, acquisitionGeneration, signal } = request + const { startup } = exited?.initialization ?? {} + if (exited?.acquisitionGeneration === acquisitionGeneration && startup && !startup.opened()) { + const ended = await waitForPromiseWithSignal( + startup.race(new Promise(() => {}), signal).catch((error: unknown) => error), + signal + ) + return claudeStartFailure(ended) + } + return ( + endings.get(sessionId, acquisitionGeneration) ?? + claudeStartFailure(new Error(`no claude session for ${sessionId}`)) + ) +} diff --git a/src/main/codex/app-server/codex-app-server-connection-exit-order.test.ts b/src/main/codex/app-server/codex-app-server-connection-exit-order.test.ts index 6a863610562f..91ab1fe9d808 100644 --- a/src/main/codex/app-server/codex-app-server-connection-exit-order.test.ts +++ b/src/main/codex/app-server/codex-app-server-connection-exit-order.test.ts @@ -3,6 +3,9 @@ import { PassThrough } from 'node:stream' import { describe, expect, it, vi } from 'vitest' import type { spawnProcess } from '@orca/process-host' import { openCodexAppServerConnection } from './codex-app-server-connection' +import { CodexAppServerRequestError } from './codex-app-server-request-error' +import { codexStartFailure } from './codex-startup-outcome' +import canonicalRefusal from './fixtures/start-resume-canonical-filename-refusal.json' function stubChild() { const child = Object.assign(new EventEmitter(), { @@ -43,4 +46,37 @@ describe('Codex app-server exit order', () => { expect(exits[0]).toContain('codex crashed') await connection.close() }) + + it('settles a start with the final answer Codex wrote just before exiting', async () => { + const { child, spawnImpl } = stubChild() + const connection = await openCodexAppServerConnection( + { command: 'codex', args: ['app-server'] }, + {}, + spawnImpl + ) + child.stdin.on('data', (chunk: Buffer) => { + const asked = chunk + .toString() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) + .find((frame) => frame.method === canonicalRefusal.method) + if (!asked) { + return + } + const { id } = asked + // The exit is seen first; the answer's last chunk reaches Orca only afterwards. + child.emit('exit', 1, null) + setImmediate(() => + child.stdout.write( + `${JSON.stringify({ id, error: { code: canonicalRefusal.code, message: canonicalRefusal.message } })}\n` + ) + ) + }) + const resumed = connection.request(canonicalRefusal.method, {}).catch((error: unknown) => error) + const failure = codexStartFailure(await resumed) + expect(failure.cause).toBeInstanceOf(CodexAppServerRequestError) + expect(failure.retry).toBe('never') + await connection.close() + }) }) diff --git a/src/main/codex/app-server/codex-app-server-connection.ts b/src/main/codex/app-server/codex-app-server-connection.ts index 23bbb1b42cbb..614c70d35db6 100644 --- a/src/main/codex/app-server/codex-app-server-connection.ts +++ b/src/main/codex/app-server/codex-app-server-connection.ts @@ -5,6 +5,7 @@ import { createClaudeChildTreeReaper } from '../../claude/claude-agent-sdk-exit- import type { ProviderProcessLaunch } from '../../provider-process/provider-process-launch' import { buildCodexAppServerExitError } from './codex-app-server-exit-error' import { withProviderSpawnAbsence } from '../../provider-process/provider-executable-missing' +import { afterProviderOutputDrains } from '../../provider-process/provider-exit-output-drain' import { initializeCodexAppServerConnection } from './codex-app-server-handshake' import { CodexAppServerHandshakeExitUnprovenError } from './codex-app-server-handshake-exit-proof' import { @@ -48,8 +49,8 @@ async function connectCodexAppServer( // Several listeners can report one failure; preserve its first, specific cause. let terminalError: Error | null = null - function buildExitError(cause?: Error): Error { - const error = buildCodexAppServerExitError(managed.stderrTail(), cause) + function buildExitError(cause?: Error, protocolFailure = false): Error { + const error = buildCodexAppServerExitError(managed.stderrTail(), cause, protocolFailure) return withProviderSpawnAbsence(error, managed.spawnAbsence) } @@ -57,16 +58,22 @@ async function connectCodexAppServer( handlers, writeResponse, onProtocolFailure: (error) => { - handleUnexpectedEnd(error) + handleUnexpectedEnd(error, true) void managed.close() } }) - // Fail pending calls immediately, but notify the owner only once root exit is proven. - function handleUnexpectedEnd(cause?: Error): void { + // Pending calls fail once output drained, so an answer written just before exit still settles + // its call; the owner is notified only once root exit is proven. + function handleUnexpectedEnd(cause?: Error, protocolFailure = false): void { if (!terminalError) { - terminalError = buildExitError(cause) - dispatcher.failPending(terminalError) + const error = buildExitError(cause, protocolFailure) + terminalError = error + if (protocolFailure) { + dispatcher.failPending(error) + } else { + afterProviderOutputDrains(child.stdout, () => dispatcher.failPending(error), child) + } } if (managed.rootVerdict === 'exited' && closesInFlight === 0 && !exitReported) { exitReported = true @@ -105,7 +112,7 @@ async function connectCodexAppServer( } }, onFatal: (error) => { - handleUnexpectedEnd(error) + handleUnexpectedEnd(error, true) void managed.close() } }) diff --git a/src/main/codex/app-server/codex-app-server-exit-error.ts b/src/main/codex/app-server/codex-app-server-exit-error.ts index f4e04cdc936f..edb95b3ddf8b 100644 --- a/src/main/codex/app-server/codex-app-server-exit-error.ts +++ b/src/main/codex/app-server/codex-app-server-exit-error.ts @@ -9,7 +9,22 @@ import { providerStderrForDisplay } from '../../provider-process/provider-spawn- const EXIT_DETAIL_MAX_CHARS = 400 -export function buildCodexAppServerExitError(stderrTail: string, cause?: Error): Error { +/** The transport ended. Protocol failures are Orca refusing what Codex wrote, so not an exit. */ +export class CodexAppServerExitError extends Error { + constructor( + message: string, + readonly protocolFailure = false + ) { + super(message) + this.name = 'CodexAppServerExitError' + } +} + +export function buildCodexAppServerExitError( + stderrTail: string, + cause?: Error, + protocolFailure = false +): Error { const tail = providerStderrForDisplay(stderrTail).trim().slice(0, EXIT_DETAIL_MAX_CHARS) // The tail is Codex's own stderr: a log, kept behind Details. const diagnostic = providerDiagnostic(tail, 'log') @@ -22,6 +37,9 @@ export function buildCodexAppServerExitError(stderrTail: string, cause?: Error): ) } const detail = cause ? `: ${cause.message}` : tail ? `: ${tail}` : '' - const error = new Error(`codex app-server connection ended${detail}`) + const error = new CodexAppServerExitError( + `codex app-server connection ended${detail}`, + protocolFailure + ) return cause ? error : withProviderDiagnostic(error, diagnostic) } diff --git a/src/main/codex/app-server/codex-startup-failure.ts b/src/main/codex/app-server/codex-startup-outcome.ts similarity index 50% rename from src/main/codex/app-server/codex-startup-failure.ts rename to src/main/codex/app-server/codex-startup-outcome.ts index ed67510a5913..55b420c615c3 100644 --- a/src/main/codex/app-server/codex-startup-failure.ts +++ b/src/main/codex/app-server/codex-startup-outcome.ts @@ -1,4 +1,39 @@ -import type { CodexAppServerRequestError } from './codex-app-server-request-error' +import { + agentSessionAcquireFailure, + agentSessionStartFailure, + type AgentSessionStartAnswer, + type AgentSessionStartFailure +} from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { CodexAppServerRequestError } from './codex-app-server-request-error' +import { CodexAppServerUnsupportedError } from './codex-app-server-session' +import { CodexAppServerExitError } from './codex-app-server-exit-error' +import { CodexAppServerFrameSizeError } from './codex-app-server-frame-size-error' + +/** Codex's answer to a start request is final unless Codex declared it temporary; its transport + * ending with no answer once output drained is an unanswered start. */ +export const codexStartupAnswer: AgentSessionStartAnswer = (cause) => { + if (cause instanceof CodexAppServerRequestError) { + return temporaryCodexStartupAnswer(cause) ? 'temporary' : 'final' + } + if ( + cause instanceof CodexAppServerUnsupportedError || + cause instanceof CodexAppServerFrameSizeError + ) { + return 'final' + } + if (cause instanceof CodexAppServerExitError) { + return cause.protocolFailure ? 'final' : 'exited' + } + return undefined +} + +export function codexStartFailure(error: unknown, exited = false): AgentSessionStartFailure { + return agentSessionStartFailure(error, codexStartupAnswer, exited) +} + +export function codexAcquireFailure(error: unknown): unknown { + return agentSessionAcquireFailure(error, codexStartupAnswer) +} /** Only declarations whose provider contract permits replay of a start. */ export function temporaryCodexStartupAnswer(error: CodexAppServerRequestError): boolean { diff --git a/src/main/codex/codex-scripted-child.test-fixture.ts b/src/main/codex/codex-scripted-child.test-fixture.ts index 3b64304a5d32..c49ea26d4d18 100644 --- a/src/main/codex/codex-scripted-child.test-fixture.ts +++ b/src/main/codex/codex-scripted-child.test-fixture.ts @@ -6,6 +6,7 @@ import type { ScriptedAgentChild } from '../runtime/structured-agent-scripted-ch import type { CodexAppServerConnection } from './app-server/codex-app-server-connection-types' import type { openCodexAppServerConnection } from './app-server/codex-app-server-connection' import { CodexAppServerRequestError } from './app-server/codex-app-server-request-error' +import { CodexAppServerExitError } from './app-server/codex-app-server-exit-error' import { CodexAppServerTimeoutError } from './app-server/codex-app-server-session' import { codexTurnLifecycleFake } from './codex-turn-lifecycle-fake' import { withProviderRequestTimeout } from '../provider-process/provider-request-timeout' @@ -160,7 +161,11 @@ export const codexScriptedChild = ( } exitNewest = (message) => { ended = true - const error = withProviderDiagnostic(new Error(message), providerDiagnostic(message, 'log')) + // As the real connection: an exit fails the pending calls with its own exit error. + const error = withProviderDiagnostic( + new CodexAppServerExitError(message), + providerDiagnostic(message, 'log') + ) gate.reject(error) handlers.onExit?.(error, { expected: false }) } diff --git a/src/main/codex/codex-structured-session-acquire.ts b/src/main/codex/codex-structured-session-acquire.ts index 7d95b68a0641..f821002be50d 100644 --- a/src/main/codex/codex-structured-session-acquire.ts +++ b/src/main/codex/codex-structured-session-acquire.ts @@ -40,6 +40,7 @@ import type { CodexStructuredSessionTeardown } from './codex-structured-session- import type { CodexStructuredNotificationRetry } from './codex-structured-notification-retry' import type { deliverCodexServerRequest } from './codex-structured-provider-events' import { codexAcquisitionNotificationHandler } from './codex-structured-acquisition-notification' +import { codexStartFailure } from './app-server/codex-startup-outcome' export async function acquireCodexStructuredSession(input: { input: StructuredAgentSessionAcquireInput @@ -163,6 +164,15 @@ export async function acquireCodexStructuredSession(input: { onExit: (error, exit) => { acquisition.appliedSettings.end() try { + // Orca's own close or cancel ends no start; Codex ending on its own does. + if (attempt.acquisitionGeneration && !exit?.expected && !attempt.cancelled) { + acquireInput.events?.observeRootExit?.() + acquisitions.endings.record( + sessionId, + attempt.acquisitionGeneration, + codexStartFailure(error) + ) + } if (attempt.acquisitionGeneration && !sessions.has(sessionId)) { attempt.exitProven = true attempt.exitError = error diff --git a/src/main/codex/codex-structured-session-adapter-fixture.ts b/src/main/codex/codex-structured-session-adapter-fixture.ts index 7e1738d7aca3..29017e372bcf 100644 --- a/src/main/codex/codex-structured-session-adapter-fixture.ts +++ b/src/main/codex/codex-structured-session-adapter-fixture.ts @@ -21,6 +21,7 @@ import type { } from './codex-structured-session-state' import { codexProviderHandle } from '../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' +import { withObservedProviderExit } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' const startLinks = new WeakMap< CodexStructuredSessionAdapter, @@ -90,13 +91,34 @@ type FakeConnection = Omit & { closeCount: number } +const STARTUP_METHODS = new Set(['thread/start', 'thread/resume', 'thread/read']) + export function fakeCodex(routes: Record = {}): { connections: FakeConnection[] openConnection: typeof openCodexAppServerConnection routes: Record } { const connections: FakeConnection[] = [] - const openConnection: typeof openCodexAppServerConnection = async (launch, handlers = {}) => { + const openConnection: typeof openCodexAppServerConnection = async ( + launch, + adapterHandlers = {} + ) => { + // As the real connection: a spontaneous exit also fails the start's calls still waiting. + const waiting = new Set<(error: Error) => void>() + const handlers: CodexAppServerConnectionHandlers = { + ...adapterHandlers, + onExit: (error, exit) => { + // The real connection's own exit error reads as the agent's exit, never a refusal. + adapterHandlers.onExit?.(exit?.expected ? error : withObservedProviderExit(error), exit) + if (exit?.expected) { + return + } + for (const fail of waiting) { + fail(error) + } + waiting.clear() + } + } const connection: FakeConnection = { launch, handlers, @@ -108,7 +130,14 @@ export function fakeCodex(routes: Record = {}): { request: async (method, params) => { connection.calls.push({ method, params }) const route = routes[method] - return route ? route(params) : {} + const answer = route ? route(params) : {} + // Turn calls keep the answers their tests script; a start's calls end with the child. + if (!(answer instanceof Promise) || !STARTUP_METHODS.has(method)) { + return answer + } + const exited = new Promise((_resolve, reject) => waiting.add(reject)) + void exited.catch(() => {}) + return Promise.race([answer, exited]) }, notify: () => {}, respond: (id, result) => connection.replies.push({ id, result }), diff --git a/src/main/codex/codex-structured-session-adapter.ts b/src/main/codex/codex-structured-session-adapter.ts index eb9e055633e8..7d02fe579fef 100644 --- a/src/main/codex/codex-structured-session-adapter.ts +++ b/src/main/codex/codex-structured-session-adapter.ts @@ -48,6 +48,8 @@ import { cancelCodexStructuredTurn } from './codex-structured-prompt-ownership' import { renameCodexProviderSession } from './codex-provider-session-rename' +import { codexAcquireFailure } from './app-server/codex-startup-outcome' +import { openCodexStart, recoverCodexRewindAtStart } from './codex-structured-start-verdict' export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdapter { private readonly sessions = new Map() @@ -111,15 +113,12 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap this.handleUnhandledFrame(sessionId, kind, payload), forceCloseUnexpected: (sessionId, fence, acquisitionGeneration, reason) => this.teardown.forceCloseUnexpected(sessionId, fence, acquisitionGeneration, reason) + }).catch((error: unknown) => { + throw codexAcquireFailure(error) }) - open = (input: StructuredAgentSessionProviderOpenInput) => { - const attempt = this.acquisitions.get(input.sessionId) - if (!attempt?.open) { - return Promise.reject(new Error(`no spawned codex child for session ${input.sessionId}`)) - } - return attempt.open(input) - } + open = (input: StructuredAgentSessionProviderOpenInput) => + openCodexStart(this.acquisitions, input) /** Buffers pre-publication events and drops events from superseded children. */ private deliver( @@ -284,7 +283,10 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap codexRewind.rewindCodexSession(this.session(input.sessionId), input, this.deps.requestTimeoutMs) recoverRewind: NonNullable = (input) => - codexRewind.recoverCodexRewind(this.session(input.sessionId), input, this.deps.requestTimeoutMs) + recoverCodexRewindAtStart(this.sessions, input, this.deps.requestTimeoutMs) + + startupEnding = (sessionId: string, generation: string) => + this.acquisitions.endings.get(sessionId, generation) compact: NonNullable = (input) => compactCodexStructuredSession(this.session(input.sessionId), input, this.deps.requestTimeoutMs) diff --git a/src/main/codex/codex-structured-session-state.ts b/src/main/codex/codex-structured-session-state.ts index 8c74bfc0ac5a..79da3c1b73cd 100644 --- a/src/main/codex/codex-structured-session-state.ts +++ b/src/main/codex/codex-structured-session-state.ts @@ -12,6 +12,7 @@ import { cancelProcessAcquisition } from '@orca/process-host/cancel-process-acqu import type { CodexAppServerConnection } from './app-server/codex-app-server-connection-types' import type { openCodexAppServerConnection } from './app-server/codex-app-server-connection' import { CodexAcquisitionWindow } from './codex-structured-acquisition-window' +import { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' import { createCodexTurnOpenWaits, type CodexTurnOpenWaits @@ -255,6 +256,8 @@ export function createCodexAcquisitionAttempt(): CodexAcquisitionAttempt { export class CodexAcquisitionRegistry { private readonly attempts = new Map() + /** Children that ended on their own before they were ready answer for it, open included. */ + readonly endings = new AgentSessionStartEndings() private closing = false get size(): number { @@ -269,6 +272,7 @@ export class CodexAcquisitionRegistry { throw new Error('codex structured session adapter is closing') } const previousAttempt = this.attempts.get(sessionId) + this.endings.forget(sessionId) const attempt = createCodexAcquisitionAttempt() this.attempts.set(sessionId, attempt) return { previousAttempt, attempt } @@ -315,6 +319,7 @@ export class CodexAcquisitionRegistry { close(): void { this.closing = true + this.endings.clear() } } diff --git a/src/main/codex/codex-structured-start-verdict.ts b/src/main/codex/codex-structured-start-verdict.ts new file mode 100644 index 000000000000..c6a6e74bcd78 --- /dev/null +++ b/src/main/codex/codex-structured-start-verdict.ts @@ -0,0 +1,44 @@ +import type { StructuredAgentSessionAdapter } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { StructuredAgentSessionProviderOpenInput } from '../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' +import { codexStartFailure } from './app-server/codex-startup-outcome' +import { recoverCodexRewind } from './codex-structured-rewind' +import { + requireLiveCodexSession, + type CodexAcquisitionRegistry, + type CodexSession +} from './codex-structured-session-state' + +/** A start's open answers with Codex's verdict; a child that already ended answers with how it + * ended, and an answer its own exit followed is still worded as that exit. */ +export function openCodexStart( + acquisitions: CodexAcquisitionRegistry, + input: StructuredAgentSessionProviderOpenInput +): ReturnType { + const attempt = acquisitions.get(input.sessionId) + const opening = attempt?.open + ? attempt.open(input) + : Promise.reject( + acquisitions.endings.get(input.sessionId, input.acquisitionGeneration) ?? + new Error(`no spawned codex child for session ${input.sessionId}`) + ) + return opening.catch((error: unknown) => { + throw codexStartFailure(error, attempt?.exitProven === true && !attempt.cancelled) + }) +} + +/** A start reads this before it is ready, so its failure carries Codex's verdict. */ +export async function recoverCodexRewindAtStart( + sessions: Map, + input: Parameters>[0], + timeoutMs?: number +): ReturnType> { + try { + return await recoverCodexRewind( + requireLiveCodexSession(sessions, input.sessionId), + input, + timeoutMs + ) + } catch (error) { + throw codexStartFailure(error) + } +} diff --git a/src/main/jsonl-rpc/agent-connection.test.ts b/src/main/jsonl-rpc/agent-connection.test.ts index c555aa90b059..bf03c2a5633e 100644 --- a/src/main/jsonl-rpc/agent-connection.test.ts +++ b/src/main/jsonl-rpc/agent-connection.test.ts @@ -146,6 +146,8 @@ describe('JSON-lines RPC process ownership', () => { const { connection, child } = fixture({ onExit, onClose }, { exitOnEnd: false }) const pending = expect(connection.request('prompt')).rejects.toThrow('stream closed') child.stdout.emit('end') + // Output ending first waits the drain bound for the exit that says how the agent ended. + await vi.advanceTimersByTimeAsync(1_000) await pending expect(connection.closed).toBe(true) expect(connection.rootVerdict).toBe('live') diff --git a/src/main/jsonl-rpc/agent-connection.ts b/src/main/jsonl-rpc/agent-connection.ts index 7c0022818424..a5f58a27d36a 100644 --- a/src/main/jsonl-rpc/agent-connection.ts +++ b/src/main/jsonl-rpc/agent-connection.ts @@ -10,6 +10,10 @@ import { type ProviderProcessExit } from '../provider-process/managed-provider-process' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' +import { + afterProviderOutputDrains, + PROVIDER_EXIT_OUTPUT_DRAIN_MS +} from '../provider-process/provider-exit-output-drain' import type { ProviderProcessCloseResult } from '../provider-process/provider-process-close' import { withProviderSpawnAbsence, @@ -29,6 +33,14 @@ export type JsonlRpcAgentConnectionOptions = JsonlRpcPeerHandlers & { onExit?: (error: Error, context: { expected: boolean; exit: ProviderProcessExit }) => void } +/** The agent process ended; pending calls fail with it only once its output drained. */ +export class JsonlRpcAgentExitError extends Error { + constructor(message: string) { + super(message) + this.name = 'JsonlRpcAgentExitError' + } +} + /** The owning runtime keeps process exit evidence separate from transport closure. */ export class JsonlRpcAgentConnection { private readonly managed: ManagedProviderProcess @@ -36,6 +48,8 @@ export class JsonlRpcAgentConnection { private closing = false private finishObservedExit?: () => void private streamExitTimer?: ReturnType + private inputEndTimer?: ReturnType + private inputEnded = false constructor( launch: ProviderProcessLaunch, @@ -53,6 +67,16 @@ export class JsonlRpcAgentConnection { managed.child.stdin, { ...options, + // Output often ends just before the exit whose report says how the agent ended. + onInputEnd: (finish) => { + this.inputEnded = true + if (this.closing || managed.rootVerdict === 'exited') { + finish() + } else { + this.inputEndTimer = setTimeout(finish, PROVIDER_EXIT_OUTPUT_DRAIN_MS) + this.inputEndTimer.unref() + } + }, onClose: (error) => { const fail = (): void => { if (this.closing || managed.rootVerdict === 'exited') { @@ -87,9 +111,10 @@ export class JsonlRpcAgentConnection { managed.child.stderr.on('error', this.onError) managed.onExit((exit) => { clearTimeout(this.streamExitTimer) + clearTimeout(this.inputEndTimer) const error = withProviderSpawnAbsence( withProviderDiagnostic( - new Error( + new JsonlRpcAgentExitError( exit.processless ? 'Agent process could not start' : `Agent process exited (code ${exit.code ?? 'none'}, signal ${exit.signal ?? 'none'})` @@ -99,17 +124,7 @@ export class JsonlRpcAgentConnection { managed.spawnAbsence ) this.peer.finishOnInputEnd(error) - const stdout = managed.child.stdout - let finished = false - let timer: ReturnType | undefined const finish = (): void => { - if (finished) { - return - } - finished = true - clearTimeout(timer) - stdout.removeListener('end', finish) - stdout.removeListener('close', finish) this.finishObservedExit = undefined this.peer.close(error) managed.child.removeListener('error', this.onError) @@ -119,21 +134,15 @@ export class JsonlRpcAgentConnection { this.diagnose('Agent exit observer failed') } } - this.finishObservedExit = finish - if ( - this.closing || - exit.processless || - this.peer.closed || - stdout.destroyed || - stdout.readableEnded - ) { + if (this.closing || exit.processless || this.peer.closed || this.inputEnded) { finish() } else { - stdout.once('end', finish) - stdout.once('close', finish) // A descendant can inherit stdout after the provider root exits. - timer = setTimeout(finish, 1_000) - timer.unref() + this.finishObservedExit = afterProviderOutputDrains( + managed.child.stdout, + finish, + managed.child + ) } }) } @@ -158,8 +167,9 @@ export class JsonlRpcAgentConnection { return this.managed.lastCloseResult } - onExit(listener: () => void): void { - this.managed.onExit(listener) + /** `requested`: Orca was closing the agent when it exited. */ + onExit(listener: (requested?: boolean) => void): void { + this.managed.onExit(() => listener(this.closing)) } request( @@ -186,6 +196,7 @@ export class JsonlRpcAgentConnection { close(error?: Error): Promise { this.closing = true clearTimeout(this.streamExitTimer) + clearTimeout(this.inputEndTimer) this.finishObservedExit?.() this.peer.close(error) return this.managed.close() diff --git a/src/main/jsonl-rpc/peer-reply.ts b/src/main/jsonl-rpc/peer-reply.ts new file mode 100644 index 000000000000..2fb209517f71 --- /dev/null +++ b/src/main/jsonl-rpc/peer-reply.ts @@ -0,0 +1,70 @@ +import { z } from 'zod' + +const responseSchema = z.looseObject({ + type: z.literal('response'), + id: z.string().optional(), + command: z.string(), + success: z.boolean(), + data: z.unknown().optional(), + error: z.string().optional() +}) + +export class JsonlRpcResponseError extends Error { + constructor( + readonly command: string, + message: string + ) { + super(message) + this.name = 'JsonlRpcResponseError' + } +} + +export type JsonlRpcPendingRequest = { + command: string + resolve: (value: unknown) => void + reject: (error: Error) => void + timer?: ReturnType +} + +/** Settles the outstanding call a record answers; false when the record is no reply to one. */ +export function settleJsonlRpcReply( + record: { type: string; id?: unknown }, + pending: Map, + rejectPending: (id: string, error: Error) => void +): boolean { + if ( + record.type !== 'response' && + record.type !== 'extension_ui_request' && + typeof record.id === 'string' && + pending.has(record.id) && + (Object.hasOwn(record, 'command') || Object.hasOwn(record, 'success')) + ) { + rejectPending(record.id, new Error('Invalid agent RPC response')) + return true + } + if (record.type !== 'response' || typeof record.id !== 'string') { + return false + } + const waiting = pending.get(record.id) + if (!waiting) { + return true + } + const response = responseSchema.safeParse(record) + if (!response.success || response.data.command !== waiting.command) { + rejectPending(record.id, new Error('Invalid agent RPC response')) + return true + } + pending.delete(record.id) + clearTimeout(waiting.timer) + if (response.data.success) { + waiting.resolve(response.data.data) + } else { + waiting.reject( + new JsonlRpcResponseError( + waiting.command, + response.data.error ?? `Agent rejected ${waiting.command}` + ) + ) + } + return true +} diff --git a/src/main/jsonl-rpc/peer.ts b/src/main/jsonl-rpc/peer.ts index f08656cae757..74a1005c8603 100644 --- a/src/main/jsonl-rpc/peer.ts +++ b/src/main/jsonl-rpc/peer.ts @@ -6,22 +6,16 @@ import { } from '../../shared/main-process-ndjson-framer' import { detachProviderStreamErrorHandler } from '../provider-process/provider-stdio-error-boundary' import { ProviderStdioWriteQueue } from '../provider-process/provider-stdio-write-queue' +import { ProviderOutputEnd } from '../provider-process/provider-output-end' import { jsonlRpcRequestTimeout, resolveJsonlRpcPeerOptions, type JsonlRpcPeerOptions } from './peer-limits' import { settleOversizedJsonlRecord, type JsonlRpcRequestOptions } from './peer-oversized-record' +import { settleJsonlRpcReply, type JsonlRpcPendingRequest } from './peer-reply' const recordSchema = z.looseObject({ type: z.string() }) -const responseSchema = z.looseObject({ - type: z.literal('response'), - id: z.string().optional(), - command: z.string(), - success: z.boolean(), - data: z.unknown().optional(), - error: z.string().optional() -}) export type JsonlRpcRecord = z.infer export class JsonlRpcStreamClosedError extends Error { @@ -30,15 +24,7 @@ export class JsonlRpcStreamClosedError extends Error { this.name = 'JsonlRpcStreamClosedError' } } -export class JsonlRpcResponseError extends Error { - constructor( - readonly command: string, - message: string - ) { - super(message) - this.name = 'JsonlRpcResponseError' - } -} +export { JsonlRpcResponseError } from './peer-reply' export type JsonlRpcPeerHandlers = { /** Includes id-less deferred prompt acknowledgements; dialect failures are fatal to preserve lifecycle evidence. */ onRecord?: (record: JsonlRpcRecord) => void @@ -46,29 +32,26 @@ export type JsonlRpcPeerHandlers = { onOversizedEvent?: (type: string, prefix: string) => void onClose?: (error: Error) => void onDiagnostic?: (message: string) => void -} -type PendingRequest = { - command: string - resolve: (value: unknown) => void - reject: (error: Error) => void - timer?: ReturnType + /** Holds the close at the end of the agent's output, as for its exit report; `finish` closes. */ + onInputEnd?: (finish: () => void) => void } /** Transport only: a dialect owns turns, dialogs, and session handles. */ export class JsonlRpcPeer { private readonly limits: Required - private readonly pending = new Map() + private readonly pending = new Map() private readonly writer: ProviderStdioWriteQueue private readonly framer: ReturnType private nextId = 0 private nextOptionalId = 0 private terminalError?: Error private inputEndError?: Error + private readonly outputEnd: ProviderOutputEnd private readingPaused = false constructor( private readonly input: Readable, - private readonly output: Writable, + output: Writable, private readonly handlers: JsonlRpcPeerHandlers = {}, options: JsonlRpcPeerOptions = {} ) { @@ -76,7 +59,7 @@ export class JsonlRpcPeer { this.writer = new ProviderStdioWriteQueue( output, this.limits.maxQueuedWriteBytes, - (error) => this.close(error), + (error) => this.endOutput(error), { capacity: () => new Error('JSON-lines RPC write queue capacity exceeded'), closed: () => new Error('JSON-lines RPC output is not writable') @@ -111,11 +94,15 @@ export class JsonlRpcPeer { input.on('end', this.onEnd) input.on('close', this.onEnd) input.on('error', this.onError) - output.on('finish', this.onEnd) - output.on('close', this.onEnd) - output.on('error', this.onError) - if (input.destroyed || input.readableEnded || output.destroyed || !output.writable) { + this.outputEnd = new ProviderOutputEnd( + output, + (error) => this.endOutput(error), + () => this.inputEndError ?? new JsonlRpcStreamClosedError() + ) + if (input.destroyed || input.readableEnded) { this.onEnd() + } else if (output.destroyed || !output.writable) { + this.endOutput(new JsonlRpcStreamClosedError()) } } @@ -153,8 +140,8 @@ export class JsonlRpcPeer { params: Record = {}, options: JsonlRpcRequestOptions = {} ): Promise { - if (this.terminalError) { - return Promise.reject(this.terminalError) + if (this.terminalError ?? this.outputEnd.error) { + return Promise.reject(this.terminalError ?? this.outputEnd.error) } if (this.pending.size >= this.limits.maxPendingRequests) { return Promise.reject(new Error('JSON-lines RPC pending request capacity exceeded')) @@ -190,7 +177,10 @@ export class JsonlRpcPeer { this.pending.set(id, { command, resolve, reject, timer }) void this.send({ ...params, type: command, id }, controller.signal).catch( (error: unknown) => { - this.rejectPending(id, error instanceof Error ? error : new Error(String(error))) + // A write lost to the agent no longer reading leaves the call to the output drain. + if (!this.outputEnd.error) { + this.rejectPending(id, error instanceof Error ? error : new Error(String(error))) + } } ) }) @@ -217,9 +207,7 @@ export class JsonlRpcPeer { this.input.removeListener('end', this.onEnd) this.input.removeListener('close', this.onEnd) detachProviderStreamErrorHandler(this.input, this.onError) - this.output.removeListener('finish', this.onEnd) - this.output.removeListener('close', this.onEnd) - detachProviderStreamErrorHandler(this.output, this.onError) + this.outputEnd.detach() this.framer.reset() this.writer.close(error) for (const id of this.pending.keys()) { @@ -239,8 +227,25 @@ export class JsonlRpcPeer { this.close(error instanceof Error ? error : new Error(String(error))) } } - private readonly onEnd = (): void => - this.close(this.inputEndError ?? new JsonlRpcStreamClosedError()) + private readonly onEnd = (): void => { + const finish = (): void => + this.close(this.inputEndError ?? this.outputEnd.error ?? new JsonlRpcStreamClosedError()) + if (this.inputEndError || !this.handlers.onInputEnd) { + finish() + } else { + this.handlers.onInputEnd(finish) + } + } + + private endOutput(error: Error): void { + if (this.terminalError || !this.outputEnd.begin(error)) { + return + } + this.writer.close(error) + this.outputEnd.settle(this.pending.size > 0, this.input, () => + this.close(this.inputEndError ?? error) + ) + } private readonly onError = (error: Error): void => this.close(error) private diagnose(message: string): void { @@ -275,37 +280,9 @@ export class JsonlRpcPeer { return } const record = parsed.data - if ( - record.type !== 'response' && - record.type !== 'extension_ui_request' && - typeof record.id === 'string' && - this.pending.has(record.id) && - (Object.hasOwn(record, 'command') || Object.hasOwn(record, 'success')) - ) { - this.rejectPending(record.id, new Error('Invalid agent RPC response')) - return - } - if (record.type === 'response' && typeof record.id === 'string') { - const pending = this.pending.get(record.id) - if (!pending) { - return - } - const response = responseSchema.safeParse(record) - if (!response.success || response.data.command !== pending.command) { - this.rejectPending(record.id, new Error('Invalid agent RPC response')) - return - } - this.pending.delete(record.id) - clearTimeout(pending.timer) - if (response.data.success) { - pending.resolve(response.data.data) - } else { - pending.reject( - new JsonlRpcResponseError( - pending.command, - response.data.error ?? `Agent rejected ${pending.command}` - ) - ) + if (settleJsonlRpcReply(record, this.pending, (id, error) => this.rejectPending(id, error))) { + if (this.outputEnd.error && this.pending.size === 0) { + this.close(this.inputEndError ?? this.outputEnd.error) } return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts index 19f83474642d..1e413c478fc9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts @@ -156,6 +156,12 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi this.liveOwnerOrNull(sessionId)?.holdsLiveProviderProcess?.(sessionId, acquisitionGeneration) ?? false + // An ended child's route may already be stopped; its adapter still holds the ending. + startupEnding = (sessionId: string, acquisitionGeneration: string) => + this.routes + .get(sessionId) + ?.registration.adapter.startupEnding?.(sessionId, acquisitionGeneration) + awaitStoppedRequestEnd = async (sessionId: string, stoppedAt: number) => this.liveOwnerOrNull(sessionId)?.awaitStoppedRequestEnd?.(sessionId, stoppedAt) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index 1582e5192331..447262a2aec9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -46,6 +46,7 @@ import type { StructuredAgentSessionAcquisitionReceipt } from './structured-agent-session-startup-attempt-contract' import type { AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' import type { StructuredAgentSessionProviderOpen, StructuredAgentSessionProviderOpenResult @@ -268,6 +269,11 @@ export type StructuredAgentSessionAdapter = StructuredAgentSessionAdapterStop & heldSends?(sessionId: string): ReadonlySet | undefined /** Prove this acquisition’s root alive for lease renewal; absent/false falls back to PID probing. */ holdsLiveProviderProcess?(sessionId: string, acquisitionGeneration: string): boolean + /** This exact acquisition's ending before the host committed it ready, as the adapter reads it. */ + startupEnding?( + sessionId: string, + acquisitionGeneration: string + ): AgentSessionStartFailure | undefined /** Undefined preserves the client catalog when the provider reports none. */ readCommands?(sessionId: string): AgentSessionSlashCommand[] | undefined atRestCommands?: StructuredAgentSessionAtRestCommands diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts index 83e2798c2f7f..d481cce8d6d0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts @@ -25,7 +25,7 @@ import { registerStructuredAgentSessionStart, structuredAgentSessionStartStillQueued } from './structured-agent-session-start-attempts' -import { retryableStructuredAgentSessionAcquisition } from './structured-agent-session-start-retry' +import { retryableAgentSessionAcquisition } from './structured-agent-session-start-failure' import { failedCreateRefusal } from './structured-agent-session-failed-create-refusal' import { startFailureDiagnosticOf } from './structured-agent-session-failure-text' import { @@ -227,7 +227,7 @@ function withDiagnostic( ...(aborted ? { aborted: true as const } : {}), ...(expired ? { expired: true as const } : {}), ...(argumentProblem ? { argumentProblem } : {}), - retry: retryableStructuredAgentSessionAcquisition(error) + retry: retryableAgentSessionAcquisition(error) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts index fbd1cbfbc2f3..901d0a83b587 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts @@ -31,6 +31,7 @@ import { import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { codexProviderHandle } from '../../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { structuredAgentSessionHeldStarts } from './structured-agent-session-held-start.test-support' const CALLER = { callerKey: 'client-1' } const EXIT_REASON = 'Claude Code is not signed in. Sign in with the Claude CLI' @@ -40,6 +41,7 @@ let store: AgentSessionRecordStore let host: StructuredAgentSessionHost let acquire: Mock let generation = 0 +const starts = structuredAgentSessionHeldStarts(() => host) async function liveReader() { const events: AgentSessionSubscribeEvent[] = [] @@ -117,6 +119,7 @@ function providerSink() { } beforeEach(async () => { + starts.clear() root = await mkdtemp(join(tmpdir(), 'orca-append-delivery-')) resetHostTestOperationIds() generation = 0 @@ -136,16 +139,17 @@ beforeEach(async () => { agents: NO_STRUCTURED_AGENTS, logger: createStructuredAgentSessionLogger(), store, - adapter: { + // The adapter answers its never-proved opens with its own child's exit. + adapter: starts.wrap({ open: () => new Promise(() => {}), - acquire, + acquire: starts.held(acquire), releaseAcquisition: vi.fn(async () => true), closeSession: vi.fn(async () => true), dispatch: vi.fn(async () => ({ state: 'admitted' as const })), cancelTurn: vi.fn(async () => ({ cancelled: true })), answerPrompt: vi.fn(async () => undefined), setOption: vi.fn(async () => undefined) - }, + }), journalDatabase: openTestJournalHostDatabase(root), claimKeyId: 'key-1', mintSpawnToken: () => `spawn-${generation + 1}`, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts index bfade9e4ca0a..949d7a38d122 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts @@ -68,7 +68,7 @@ export function endStructuredAgentSessionReleasedChild( const session = context.sessions.get(sessionId) const child = session?.child if (child) { - context.runtimeState.startupAttempts.childEnded(sessionId, child) + context.runtimeState.startupAttempts.childEnded(sessionId, child, { byOrca: true }) } if ( !session || diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-publish.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-publish.ts index e6fe5badf319..e39dc941c08b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-publish.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-publish.ts @@ -134,23 +134,24 @@ export async function publishAttachedChild( candidate.child ) } - // Apply exits observed during background startup after indexing the matching child. + // Apply exits observed during background startup after indexing the matching child; the open + // claims a starting child's outcome first, so the adapter answers for its own early exit. const exited = context.runtimeState.startFlights.takeExit(sessionId, candidate.child) const indexed = conversation.child + const opens = !!attempt.startup && candidate.child.phase !== 'ready' + if (opens) { + context.openProviderChild?.(sessionId, candidate.child) + } if (exited && indexed) { await context.endExitedChild(sessionId, indexed, structuredAgentSessionEndedEventExit(exited)) - } else if (attempt.startup) { - if (candidate.child.phase === 'ready') { - noteStructuredAgentSessionProviderStarted( - context.deps, - sessionId, - () => - context.sessions.get(sessionId)?.child === candidate.child && - candidate.child.catalogSource?.() !== false - ) - } else { - context.openProviderChild?.(sessionId, candidate.child) - } + } else if (attempt.startup && !opens) { + noteStructuredAgentSessionProviderStarted( + context.deps, + sessionId, + () => + context.sessions.get(sessionId)?.child === candidate.child && + candidate.child.catalogSource?.() !== false + ) } context.publishStatus?.(sessionId) return true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index e7360736956a..096bebfd69a7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -68,7 +68,7 @@ export type StructuredAgentSessionChildExitContext< /** Re-derive current work and wake queue without requiring a row; restate rebaselines moved fences. */ generationEnded: StructuredAgentSessionGenerationEnded wakeDelivery?: (sessionId: string) => void - startupAttempts?: Pick + startupAttempts?: Pick /** Retain exits between background-start spawn and publication. */ startFlights?: Pick abortBound?: (sessionId: string) => void @@ -148,9 +148,10 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< // Use the first death observation when settling the running turn. const observation = child.exitObservation?.() const observedAt = observation?.observedAt ?? exit.observedAt ?? context.now() - // Host phase classifies startup failure even if adapters omit the flag. + // Host phase classifies startup failure even if adapters omit the flag. A starting child's + // exit cancels nothing: the start's owner gives its outcome from the adapter's answer. const exitedDuringStartup = child.phase === 'starting' - if (child.phase === 'starting' || !expected) { + if (!exitedDuringStartup && !expected) { context.abortBound?.(sessionId) } // Personal Stop cancels startup; host Stop fails it; preserve the close initiator. @@ -187,7 +188,10 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< }) context.generationEnded(sessionId) const endChild = (): void => { - context.startupAttempts?.childEnded(sessionId, child) + context.startupAttempts?.childEnded(sessionId, child, { + byOrca: expected || !!close, + ...(close?.failure ? { failure: close.failure, reason: close.reason ?? exit.reason } : {}) + }) if (session.child === child) { session.child = null } @@ -258,7 +262,9 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< ? { exitedDuringStartup: { generation: child.generation, - flightOwned: context.startFlights?.ownsChild(sessionId, child) + owned: + context.startFlights?.ownsChild(sessionId, child) || + context.startupAttempts?.ownsOpen(sessionId, child) } } : {}), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 5ec86550cd99..22d259321736 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -3,7 +3,7 @@ import { cutClosingStructuredAgentSession, type StructuredAgentSessionConversationLifetimeHost } from './structured-agent-session-conversation-lifetime-host' -import { retryableStructuredAgentSessionStart } from './structured-agent-session-start-retry' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' // Open conversations at rest; never start agents for reads. Acquire serialization once because the queue is nonreentrant. export type { StructuredAgentSessionConversationLifetime } from './structured-agent-session-host-types' @@ -37,10 +37,7 @@ import { type WorkspaceSleepCloseResult } from './structured-agent-session-workspace-sleep-stop' import type { StructuredAgentSessionExpiredStartup } from './structured-agent-session-startup-attempt' -import { - providerStartupFailureFact, - structuredAgentSessionStartFailure -} from './structured-agent-session-failure-text' +import { structuredAgentSessionStartFailure } from './structured-agent-session-failure-text' import { oldestQueuedSubmission, recordStructuredAgentSessionStartFailure @@ -212,57 +209,55 @@ export function createStructuredAgentSessionConversationLifetime( expireStartup: ({ sessionId, child }: StructuredAgentSessionExpiredStartup): void => stopStartingChild(sessionId, child), stopStartingChild, + /** The open's failed start: a direct attach records its row; a still-indexed child stops. A + * replaced, stopped or closed child only retires its owner. */ failStartingChild: ( sessionId: string, failed: StructuredAgentSessionProviderChildIdentity, - error: unknown, + verdict: AgentSessionStartFailure, openMs: number ): Promise => serialize(sessionId, async () => { const session = sessions.get(sessionId) const child = session?.child - if (!session || !child || child.close || !sameProviderChild(child, failed)) { + const indexed = !!child && sameProviderChild(child, failed) + const ended = session?.lastEndedChild + const exited = + !child && !!ended && sameProviderChild(ended, failed) && ended.cause === 'exit' + if (!session || (indexed ? child?.close : !exited)) { return } - const failure = providerStartupFailureFact(error) + const failure = verdict.fact // Persisted facts cannot retain the original error; log it here. deps().logger.warn('a provider could not finish starting', { scope: 'provider-open-failed', sessionId, failure: failure.kind, ...failure.requestTimeout, + retry: verdict.retry, openMs, - error + error: verdict.cause }) - const { runtimeState: runtime } = host.context() - const owned = runtime.startFlights.ownsChild(sessionId, child) - if (owned) { - const retry = retryableStructuredAgentSessionStart(error) - runtime.startupAttempts.childFailed(sessionId, child, failure, retry) + if (!host.context().runtimeState.startFlights.ownsChild(sessionId, failed)) { + const record = deps().store.getRecord(sessionId) + const wording = structuredAgentSessionFailureWordsContext(record) + await recordStructuredAgentSessionStartFailure( + { journal: session.journal, fence: failed.fence }, + { + startKey: failed.generation, + ...structuredAgentSessionStartFailure({ failure }, wording) + }, + oldestQueuedSubmission(session)?.clientMessageId ?? null + ).catch((writeError: unknown) => + deps().logger.warn('recording a failed provider open failed', { + scope: 'provider-open-failure', + sessionId, + error: writeError + }) + ) } - try { - if (!owned) { - const record = deps().store.getRecord(sessionId) - const wording = structuredAgentSessionFailureWordsContext(record) - await recordStructuredAgentSessionStartFailure( - { journal: session.journal, fence: child.fence }, - { - startKey: child.generation, - ...structuredAgentSessionStartFailure({ failure }, wording) - }, - oldestQueuedSubmission(session)?.clientMessageId ?? null - ).catch((writeError: unknown) => - deps().logger.warn('recording a failed provider open failed', { - scope: 'provider-open-failure', - sessionId, - error: writeError - }) - ) - } - const reason = error instanceof Error ? error.message : String(error) - await stopAgent(sessionId, { cause: 'host-stop', reason, failure }) - } finally { - runtime.startupAttempts.childEnded(sessionId, child) + if (indexed) { + await stopAgent(sessionId, { cause: 'host-stop', reason: verdict.message, failure }) } }), dispose: (): void => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index a779a7c53057..d376688b1747 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -59,8 +59,9 @@ export type StructuredAgentSessionDeadGenerationInput = { showUnexpectedExitOutcome?: boolean exitFailure?: SubmissionRejectionFact failureTextContext?: AgentSessionFailureWordsContext - /** Share generation-keyed startup failure rows with delivery-loop settlement. */ - exitedDuringStartup?: { generation: string | null; flightOwned?: boolean } + /** Share generation-keyed startup failure rows with delivery-loop settlement. `owned`: a start + * flight or the child's open still owns its outcome and row. */ + exitedDuringStartup?: { generation: string | null; owned?: boolean } /** Refine translator-unverifiable rows after observed exit proof. */ exit?: StructuredAgentSessionWatchedExit unrunRejection?: SubmissionRejectionFact @@ -104,7 +105,7 @@ export function planStructuredAgentSessionDeadGeneration( // Delivery owns failure rows for sends; other startup effects need their own row. // Existing rejection wording is terminal and must remain unchanged. const recordedByDeliveryLoop = - input.exitedDuringStartup.flightOwned || + input.exitedDuringStartup.owned || input.journal.submissions?.().some(isQueuedAgentJournalSubmission) || hasStructuredAgentSessionStartFailureRow(allItems, startKey) if (!recordedByDeliveryLoop) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 87cb788588e0..62602a8e272f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -18,7 +18,10 @@ import { type StructuredAgentSessionGenerationEnded } from './structured-agent-session-child-exit' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' -import { openStructuredAgentSessionProviderChild } from './structured-agent-session-provider-open' +import { + openStructuredAgentSessionProviderChild, + type StructuredAgentSessionFailStartingChild +} from './structured-agent-session-provider-open' export class StructuredAgentSessionEventRecovery { private readonly sinkFailures = new Set() @@ -35,16 +38,7 @@ export class StructuredAgentSessionEventRecovery { now: () => number runtimeState: StructuredAgentSessionHostRuntimeState wakeDelivery: (sessionId: string) => Promise - stopStartingChild: ( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity - ) => void - failStartingChild: ( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity, - error: unknown, - openMs: number - ) => Promise + failStartingChild: StructuredAgentSessionFailStartingChild trackOpen: (open: Promise) => Promise } ) {} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index e1b9dfc3e3da..1f73c74235df 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -32,7 +32,10 @@ export type { StructuredAgentSessionLinkageJournal, StructuredAgentSessionRevisionResolver } from './structured-agent-session-journal-views' -import { StructuredAgentSessionExitObservation } from './structured-agent-session-exit-observation' +import { + StructuredAgentSessionExitObservation, + StructuredAgentSessionRootExitReceipt +} from './structured-agent-session-exit-observation' export type StructuredAgentSessionSinkAdmission = | { accepted: true } @@ -72,6 +75,9 @@ export type StructuredAgentSessionRevisionOptions = StructuredAgentSessionItemAp export type StructuredAgentSessionEventSink = StructuredAgentSessionTransitionSink & { observeExit?: StructuredAgentSessionExitObservation['record'] + /** Adapters report only their acquisition's own spontaneous root exit, never a requested close. */ + observeRootExit?: StructuredAgentSessionRootExitReceipt['observe'] + onRootExit?: StructuredAgentSessionRootExitReceipt['listen'] appendItem( identity: AgentJournalItemIdentity, body: AgentJournalItemBody, @@ -190,6 +196,7 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { onBackpressureChange?: (backpressured: boolean, state: StructuredAgentSessionSinkState) => void }): DeferredStructuredAgentSessionEventSink { const exit = new StructuredAgentSessionExitObservation() + const rootExit = new StructuredAgentSessionRootExitReceipt() const watermarks = { ...DEFAULT_WATERMARKS, ...deps.watermarks } const failed = (error: unknown): void => { deps.logger.error('writing provider events to the chat journal failed', { @@ -260,6 +267,8 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { exitObservation: () => exit, sink: { observeExit: exit.record, + observeRootExit: rootExit.observe, + onRootExit: rootExit.listen, appendItem: (identity, body, options) => void appendItem(identity, body, options), tryAppendItem: appendItem, ...resolvedAppend, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-observation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-observation.ts index 8e1287cadfa1..0e100f393cd3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-observation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-observation.ts @@ -15,3 +15,33 @@ export class StructuredAgentSessionExitObservation { this.close ??= close } } + +/** The first spontaneous root exit the adapter observed for this sink's acquisition, taken where + * the adapter sees it, before output drain, cleanup or host serialization can delay it. The + * listener's own clock times it, so adapter and host clocks never mix. */ +export class StructuredAgentSessionRootExitReceipt { + observed = false + private readonly listeners = new Set<() => void>() + + observe = (): void => { + if (this.observed) { + return + } + this.observed = true + const listeners = [...this.listeners] + this.listeners.clear() + for (const listener of listeners) { + listener() + } + } + + /** Replays an exit already observed; the returned call stops listening. */ + listen = (listener: () => void): (() => void) => { + if (this.observed) { + listener() + return () => {} + } + this.listeners.add(listener) + return () => this.listeners.delete(listener) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-held-start.test-support.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-held-start.test-support.ts index 65e0865f2536..855789c0fed3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-held-start.test-support.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-held-start.test-support.ts @@ -1,7 +1,18 @@ -import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import type { + StructuredAgentSessionAdapter, + StructuredAgentSessionLifecycleEvent +} from './structured-agent-session-adapter' +import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import { + providerStartupFailureFact, + structuredAgentSessionStartFailureFact, + withObservedProviderExit +} from './structured-agent-session-failure-text' type Acquire = StructuredAgentSessionAdapter['acquire'] type Open = StructuredAgentSessionAdapter['open'] +type EventHost = { handleAdapterEvent(event: StructuredAgentSessionLifecycleEvent): Promise } export type StructuredAgentSessionHeldStarts = { held(acquire: Acquire): Acquire @@ -10,17 +21,61 @@ export type StructuredAgentSessionHeldStarts = { clear(): void } +const EXIT_KINDS = new Set(['providerExited', 'providerStartFailed', 'startFailed']) + +/** As an adapter reads its own child's exit: a typed refusal the exit carried stays final, and an + * unanswered exit may start once more. */ +export function testAdapterStartupEnding( + failure?: SubmissionRejectionFact +): AgentSessionStartFailure { + const exited = withObservedProviderExit(new Error('provider exited before open answered')) + const final = failure && (failure.refusal !== undefined || !EXIT_KINDS.has(failure.kind)) + return new AgentSessionStartFailure( + exited, + final ? 'never' : 'safe', + failure + ? structuredAgentSessionStartFailureFact({ exit: failure }) + : providerStartupFailureFact(exited) + ) +} + +/** Opens wait until proved; as a real adapter, a child's own exit answers its waiting open. */ export function structuredAgentSessionHeldStarts( - _host: () => unknown + host: () => EventHost | undefined ): StructuredAgentSessionHeldStarts { const held = new Map void>() const waiting = new Map>() + const endings = new Map() + const ended = new Map void>() + const observed = new WeakSet() + const observe = (): void => { + const target = host() + if (!target || observed.has(target)) { + return + } + observed.add(target) + const handle = target.handleAdapterEvent.bind(target) + target.handleAdapterEvent = async (event) => { + const ending = + event.type === 'ended' && event.cause === 'unexpected-exit' + ? testAdapterStartupEnding(event.failure) + : undefined + if (ending && event.type === 'ended') { + endings.set(event.acquisitionGeneration, ending) + } + await handle(event) + if (ending && event.type === 'ended') { + ended.get(event.acquisitionGeneration)?.(ending) + } + } + } const prove: StructuredAgentSessionHeldStarts['prove'] = async (child) => { held.get(child.acquisitionGeneration)?.() await Promise.resolve() } return { held: (acquire) => async (input) => { + observe() const acquired = await acquire(input) const generation = acquired.acquisitionGeneration waiting.set(generation, new Promise((resolve) => held.set(generation, resolve))) @@ -28,15 +83,28 @@ export function structuredAgentSessionHeldStarts( }, wrap: (adapter) => { const open: Open = async (input) => { + observe() const generation = input.acquisitionGeneration - if (generation) { - await waiting.get(generation) + const ending = endings.get(generation) + if (ending) { + throw ending + } + const exit = new Promise((_resolve, reject) => ended.set(generation, reject)) + void exit.catch(() => {}) + try { + await Promise.race([waiting.get(generation), exit]) waiting.delete(generation) held.delete(generation) + return await Promise.race([adapter.open(input), exit]) + } finally { + ended.delete(generation) } - return adapter.open(input) } - return { ...adapter, open } + const startupEnding: NonNullable = ( + _sessionId, + generation + ) => endings.get(generation) + return { ...adapter, open, startupEnding } }, prove, clear: () => { @@ -45,6 +113,7 @@ export function structuredAgentSessionHeldStarts( } held.clear() waiting.clear() + endings.clear() } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts index 01e96ab8862c..477b947aeac2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts @@ -21,7 +21,10 @@ import { StructuredAgentSessionStartupAttempts, type StructuredAgentSessionExpiredStartup } from './structured-agent-session-startup-attempt' -import { StructuredAgentSessionStartupExpiredError } from './structured-agent-session-startup-attempt-contract' +import { + StructuredAgentSessionStartupContractError, + StructuredAgentSessionStartupExpiredError +} from './structured-agent-session-startup-attempt-contract' export class StructuredAgentSessionHostRuntimeState { private readonly eventSinks = new Map() @@ -50,6 +53,16 @@ export class StructuredAgentSessionHostRuntimeState { if (expired.child !== null) { onStartupExpired?.(expired) } + }, + // The exited child still ends through its own exit; only the start's waits are cancelled. + breach: ({ sessionId, attemptId, agent }) => { + deps.logger.warn('an agent adapter gave no startup outcome after its process exited', { + scope: 'startup-contract', + sessionId, + attemptId, + agent + }) + this.acquireAborts.abort(sessionId, new StructuredAgentSessionStartupContractError(agent)) } }) this.leaseRenewer = new StructuredAgentSessionLeaseRenewer({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index b2025f322cc9..5d4ca7bc4bc1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -115,7 +115,6 @@ export class StructuredAgentSessionHost { now: () => this.now(), runtimeState: this.runtimeState, wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), - stopStartingChild: (...args) => this.lifetime.stopStartingChild(...args), failStartingChild: (...args) => this.lifetime.failStartingChild(...args), trackOpen: (open) => this.tasks.trackAttach(open) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts index 26e66bfacb0f..2680f14687e2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts @@ -12,6 +12,11 @@ import { hostTestState } from './structured-agent-session-host-test-harness' import { HOST_TEST_SESSION, hostTestMessage } from './structured-agent-session-host-test-data' +import canonicalRefusal from '../../codex/app-server/fixtures/start-resume-canonical-filename-refusal.json' +import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' +import { agentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' +import { structuredAgentSessionStartFailureRowIdentity } from '../../../shared/agent-session/failures/structured-agent-session-start-failure-row-key' const OPENED: StructuredAgentSessionProviderOpenResult = { reportedOptions: { model: 'gpt-test' } @@ -78,3 +83,31 @@ it('journals a typed writer refusal and cleans up the real starting child', asyn expect(close).toHaveBeenCalledOnce() expect(dispatch).not.toHaveBeenCalled() }) + +it("records one standalone row in the start's words when a direct attach fails with nothing queued", async () => { + const { host, open, close, dispatch, pending } = awaitOpen() + await attach() + await vi.waitFor(() => expect(open).toHaveBeenCalledOnce()) + pending.reject( + codexStartFailure( + new CodexAppServerRequestError( + 'thread/resume', + canonicalRefusal.code, + canonicalRefusal.message + ) + ) + ) + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()) + const snapshot = await host.journalSnapshot(HOST_TEST_SESSION) + expect(snapshot.items.filter((item) => item.body.kind === 'status')).toEqual([ + expect.objectContaining({ + itemId: agentJournalItemKey(structuredAgentSessionStartFailureRowIdentity('awaited-open')), + body: expect.objectContaining({ + tone: 'error', + failure: expect.objectContaining({ kind: 'startFailed' }) + }) + }) + ]) + expect(snapshot.submissions).toEqual([]) + expect(dispatch).not.toHaveBeenCalled() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts index 1828de1798a8..4a9f11d42e54 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts @@ -1,91 +1,116 @@ -import { providerStartupFailureFact } from './structured-agent-session-failure-text' -import { sameProviderChild } from './structured-agent-session-provider-child' -import { retryableStructuredAgentSessionStart } from './structured-agent-session-start-retry' import { waitForPromiseWithSignal } from '../../../shared/abort-signal-reason' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' import { settleStructuredAgentSessionProviderStarted, type StructuredAgentSessionProviderStartedContext } from './structured-agent-session-provider-started' +import { + AgentSessionStartFailure, + agentSessionStartFailure +} from './structured-agent-session-start-failure' +import { + isStructuredAgentSessionStartupContractBreach, + isStructuredAgentSessionStartupExpired +} from './structured-agent-session-startup-attempt-contract' + +export type StructuredAgentSessionFailStartingChild = ( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + failure: AgentSessionStartFailure, + openMs: number +) => Promise -/** Await protocol construction outside conversation serialization. */ +/** The one owner of a starting child's outcome: it awaits the adapter's open and the start's + * required settlement outside serialization, then gives the start its verdict once. */ export async function openStructuredAgentSessionProviderChild( context: StructuredAgentSessionProviderStartedContext & { - failStartingChild: ( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity, - error: unknown, - openMs: number - ) => Promise + failStartingChild: StructuredAgentSessionFailStartingChild }, sessionId: string, child: StructuredAgentSessionProviderChildIdentity ): Promise { - if (!context.runtimeState.startupAttempts.onClock(sessionId, child)) { + const { startupAttempts, acquireAborts } = context.runtimeState + if (!startupAttempts.own(sessionId, child)) { return } - // A start flight joins the outstanding answer after exit and retires it; a direct attach has no such owner. - const attempt = context.runtimeState.acquireAborts.begin(sessionId, false, false, () => { - const current = context.sessions.get(sessionId)?.child - return ( - current !== undefined && - current !== null && - sameProviderChild(current, child) && - !context.runtimeState.startFlights.ownsChild(sessionId, child) - ) - }) - const observation = context.runtimeState.startupAttempts.observeOpen( - sessionId, - child, - attempt.abort - ) + const attempt = acquireAborts.begin(sessionId) const openedAt = context.now() try { - attempt.signal.throwIfAborted() - const acquisitionGeneration = child.generation - if (acquisitionGeneration === null) { - throw new Error('An unopened provider child has no acquisition generation') - } - const result = await waitForPromiseWithSignal( - context.deps.adapter.open({ - sessionId, - fence: child.fence, - acquisitionGeneration, - signal: attempt.signal - }), - attempt.signal + const outcome = await openAndSettle(context, sessionId, child, attempt.signal).catch( + (error: unknown) => + attempt.signal.aborted + ? stoppedStart(attempt.signal.reason) + : error instanceof Error + ? error + : new Error(String(error)) ) - if (attempt.signal.aborted) { + if (outcome === 'ready') { return } - await settleStructuredAgentSessionProviderStarted( - context, - { ...result, sessionId, fence: child.fence, acquisitionGeneration }, - attempt.signal + if (!(outcome instanceof Error)) { + startupAttempts.released(sessionId, child) + return + } + const failure = agentSessionStartFailure(outcome) + // Queued first so the start's row precedes any send that wakes on its verdict. + const cleanup = context.failStartingChild( + sessionId, + child, + failure, + Math.max(0, context.now() - openedAt) ) - } catch (error) { - if (!attempt.signal.aborted) { - observation.failed( - providerStartupFailureFact(error), - retryableStructuredAgentSessionStart(error), + startupAttempts.failed(sessionId, child, failure) + await cleanup.catch((error: unknown) => + context.deps.logger.warn('closing a failed provider open did not finish', { + scope: 'provider-open-close', + sessionId, error - ) - } - // Aborted startup already has a Stop/close/quit/expiry owner. - if (!attempt.signal.aborted) { - const openMs = Math.max(0, context.now() - openedAt) - await context - .failStartingChild(sessionId, child, error, openMs) - .catch((closeError: unknown) => - context.deps.logger.warn('closing a failed provider open did not finish', { - scope: 'provider-open-close', - sessionId, - error: closeError - }) - ) - } + }) + ) } finally { - observation.finish() + startupAttempts.disown(sessionId, child) attempt.end() } } + +async function openAndSettle( + context: StructuredAgentSessionProviderStartedContext, + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + signal: AbortSignal +): Promise<'ready' | 'stale' | Error> { + signal.throwIfAborted() + const acquisitionGeneration = child.generation + if (acquisitionGeneration === null) { + throw new Error('An unopened provider child has no acquisition generation') + } + const result = await waitForPromiseWithSignal( + context.deps.adapter.open({ sessionId, fence: child.fence, acquisitionGeneration, signal }), + signal + ) + return waitForPromiseWithSignal( + settleStructuredAgentSessionProviderStarted( + context, + { ...result, sessionId, fence: child.fence, acquisitionGeneration }, + signal + ), + signal + ) +} + +/** Stop, close and quit settle through their own owner. The deadline and the contract guard + * already decided the start; the owner still records its row. */ +function stoppedStart(reason: unknown): 'stale' | AgentSessionStartFailure { + if (isStructuredAgentSessionStartupExpired(reason)) { + return new AgentSessionStartFailure(reason, 'never', agentSessionFailureFact('hostStopped')) + } + if (isStructuredAgentSessionStartupContractBreach(reason)) { + return new AgentSessionStartFailure( + reason, + 'never', + agentSessionFailureFact('providerStartFailed') + ) + } + return 'stale' +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started-outcome.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started-outcome.test.ts new file mode 100644 index 000000000000..98125b55a428 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started-outcome.test.ts @@ -0,0 +1,81 @@ +import { beforeEach, expect, it, vi } from 'vitest' +import canonicalRefusal from '../../codex/app-server/fixtures/start-resume-canonical-filename-refusal.json' +import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' +import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' +import { readStructuredRewindRecovery } from './structured-rewind-recovery' +import type * as RewindRecovery from './structured-rewind-recovery' +import { + settleStructuredAgentSessionProviderStarted, + type StructuredAgentSessionProviderStartedContext +} from './structured-agent-session-provider-started' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' + +vi.mock('./structured-rewind-recovery', async (importOriginal) => ({ + ...(await importOriginal()), + readStructuredRewindRecovery: vi.fn() +})) + +const SESSION = 'session-1' +const OPENED = { sessionId: SESSION, fence: 3, acquisitionGeneration: 'g-1', reportedOptions: {} } +const died = codexStartFailure(new CodexAppServerExitError('codex app-server connection ended')) +let ending: AgentSessionStartFailure | undefined + +function context(childGeneration = 'g-1'): StructuredAgentSessionProviderStartedContext { + const session = { + journal: {}, + child: { generation: childGeneration, fence: 3, phase: 'starting' } + } + const members = { + deps: { + adapter: { startupEnding: () => ending }, + store: { getRecord: () => null, getConversation: () => ({ currentFence: 3 }) }, + logger: { warn: vi.fn() } + }, + sessions: new Map([[SESSION, session]]), + serialize: (_sessionId: string, task: () => Promise) => task(), + now: () => 0, + runtimeState: { + startupAttempts: { onClock: () => true, rootExit: () => new Promise(() => {}) }, + acquireAborts: {} + }, + wakeDelivery: vi.fn() + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: settlement reads only these members before its verdict. + return members as unknown as StructuredAgentSessionProviderStartedContext +} + +beforeEach(() => { + ending = undefined + vi.mocked(readStructuredRewindRecovery).mockReset().mockResolvedValue(null) +}) + +it("keeps a required read's final answer over the child's later death", async () => { + const answer = new CodexAppServerRequestError( + 'thread/read', + canonicalRefusal.code, + canonicalRefusal.message + ) + vi.mocked(readStructuredRewindRecovery).mockRejectedValue(codexStartFailure(answer)) + ending = died + const outcome = await settleStructuredAgentSessionProviderStarted(context(), OPENED) + expect(outcome).toMatchObject({ retry: 'never', cause: answer }) +}) + +it('answers an unanswered death after a successful open with the adapter verdict', async () => { + ending = died + expect(await settleStructuredAgentSessionProviderStarted(context(), OPENED)).toBe(died) + expect(died.retry).toBe('safe') +}) + +it('retires only its own owner once the child was replaced', async () => { + expect(await settleStructuredAgentSessionProviderStarted(context('g-2'), OPENED)).toBe('stale') +}) + +it('ends the start on a reported death while its bookkeeping is still blocked', async () => { + ending = died + const blocked = context() + blocked.flushLifecycle = () => new Promise(() => {}) + blocked.runtimeState.startupAttempts.rootExit = () => Promise.resolve() + expect(await settleStructuredAgentSessionProviderStarted(blocked, OPENED)).toBe(died) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts index 1d3d7f565409..cd8059ea1559 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts @@ -1,14 +1,11 @@ import type { StructuredAgentSessionOptionsReportedEvent } from './structured-agent-session-adapter' -import type { StructuredAgentSessionProviderOpenResult } from './structured-agent-session-provider-open-contract' import type { StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession, - StructuredAgentSessionProviderChildIdentity + StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { markProviderChildStarted, - sameProviderChild, - structuredAgentSessionConversationFence + sameProviderChild } from './structured-agent-session-provider-child' import { saveStructuredAgentSessionListing, @@ -20,15 +17,16 @@ import { noteStructuredAgentSessionReportedOptions } from './structured-agent-session-ran-options' import type { StructuredAgentSessionAcquireAborts } from './structured-agent-session-acquire-aborts' -import type { StructuredAgentSessionStartFlights } from './structured-agent-session-start-flight' -import { recordAgentSessionProviderHandle } from '../../runtime/agent-session/ownership/agent-session-provider-handle-transition' -import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' import { settleUnheldStructuredAgentSessionTurns } from './structured-agent-session-unheld-turns' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' import { - applyStructuredRewindRecovery, - readStructuredRewindRecovery, - type StructuredRewindRecoveryEvidence -} from './structured-rewind-recovery' + applyRewindAtStart, + readRewindAtStart, + recordStartedHandle, + refusedStart, + type OpenedProvider +} from './structured-agent-session-start-provider-answer' +import type { StructuredRewindRecoveryEvidence } from './structured-rewind-recovery' export type StructuredAgentSessionProviderStartedContext = { deps: StructuredAgentSessionHostDeps @@ -39,216 +37,131 @@ export type StructuredAgentSessionProviderStartedContext = { runtimeState: { startupAttempts: Pick< StructuredAgentSessionStartupAttempts, - 'ready' | 'onClock' | 'readiness' | 'observeOpen' + 'ready' | 'onClock' | 'readiness' | 'own' | 'disown' | 'failed' | 'released' | 'rootExit' > - startFlights: Pick acquireAborts: Pick } /** Commits the child's buffered lifecycle events. */ flushLifecycle?: (sessionId: string) => Promise /** Release startup’s delivery barrier and drain all currently handable sends. */ wakeDelivery: (sessionId: string) => Promise - stopStartingChild: (sessionId: string, child: StructuredAgentSessionProviderChildIdentity) => void } -type OpenedProvider = StructuredAgentSessionProviderOpenResult & { - sessionId: string - fence: number - acquisitionGeneration: string -} +/** A failure is the adapter's verdict on required provider work or on this child ending before it + * was ready, or Orca refusing what the open answered; `stale` retires only this owner. */ +export type StructuredAgentSessionProviderStartedOutcome = + | 'ready' + | 'stale' + | AgentSessionStartFailure export async function settleStructuredAgentSessionProviderStarted( context: StructuredAgentSessionProviderStartedContext, event: OpenedProvider, signal?: AbortSignal -): Promise { +): Promise { // Read provider state outside serialization so sends can still be accepted. const rewind = await readRewindAtStart(context, event, signal) + // Required provider work's own verdict outranks this child's later death. + if (rewind instanceof AgentSessionStartFailure) { + return rewind + } + const child = { generation: event.acquisitionGeneration, fence: event.fence } + // Only the adapter's word on this exact acquisition says it died; absence alone says nothing. + const ended = () => + context.deps.adapter.startupEnding?.(event.sessionId, event.acquisitionGeneration) + // A death the adapter reports ends the start without waiting on its bookkeeping below. + const died = context.runtimeState.startupAttempts + .rootExit(event.sessionId, child) + .then(() => ended() ?? new Promise(() => {})) + return Promise.race([died, settleReady(context, event, rewind, ended, signal)]) +} + +function settleReady( + context: StructuredAgentSessionProviderStartedContext, + event: OpenedProvider, + rewind: StructuredRewindRecoveryEvidence, + ended: () => AgentSessionStartFailure | undefined, + signal?: AbortSignal +): Promise { + const child = { generation: event.acquisitionGeneration, fence: event.fence } // Restored history is committed and the child's own events drained before its turns are judged. - await context.flushLifecycle?.(event.sessionId) - // Follow attach publication so its proven lease has committed. - await context.serialize(event.sessionId, async () => { - signal?.throwIfAborted() - const session = context.sessions.get(event.sessionId) - const child = { generation: event.acquisitionGeneration, fence: event.fence } - // Stale or expired startup cannot lift the current child’s barrier. - if ( - !session?.child || - session.child.phase !== 'starting' || - !sameProviderChild(session.child, child) || - !context.runtimeState.startupAttempts.onClock(event.sessionId, child) - ) { - return null - } - if (!(await recordStartedHandle(context, event))) { - context.stopStartingChild(event.sessionId, child) - return null - } - if (!(await applyRewindAtStart(context, event.sessionId, session, rewind))) { - context.stopStartingChild(event.sessionId, child) - return null - } - if ( - !context.runtimeState.startupAttempts.onClock(event.sessionId, child) || - !markProviderChildStarted(session, child) - ) { - return null - } - await settleUnheldStructuredAgentSessionTurns( - session.journal, - event.sessionId, - child.fence, - event.acquisitionGeneration, - context.deps.adapter.liveTurnId?.(event.sessionId) - ).catch((error: unknown) => - context.deps.logger.warn('settling turns the agent is not running failed', { - scope: 'unheld-turns', - sessionId: event.sessionId, - error - }) - ) - const sourceChild = session.child - context.runtimeState.startupAttempts.ready(event.sessionId, child) - noteStructuredAgentSessionRuntimePermission(session.child, event.reportedOptions, true) - context.publishStatus?.(event.sessionId) - await noteStructuredAgentSessionReportedOptions( - context, - event.sessionId, - child.fence, - event.reportedOptions - ) - void context.wakeDelivery(event.sessionId) - noteStructuredAgentSessionProviderStarted( - context.deps, - event.sessionId, - () => - context.sessions.get(event.sessionId)?.child === sourceChild && - sourceChild?.catalogSource?.() !== false - ) - if (event.catalogListing) { - saveStructuredAgentSessionListing( + return Promise.resolve(context.flushLifecycle?.(event.sessionId)).then(() => + // Follow attach publication so its proven lease has committed. + context.serialize(event.sessionId, async () => { + signal?.throwIfAborted() + const session = context.sessions.get(event.sessionId) + const ending = ended() + const current = session?.child && sameProviderChild(session.child, child) ? session : null + if (!current?.child) { + return ending ?? 'stale' + } + // Stale or expired startup cannot lift the current child’s barrier. + if ( + current.child.phase !== 'starting' || + !context.runtimeState.startupAttempts.onClock(event.sessionId, child) + ) { + return 'stale' + } + if (ending) { + return ending + } + if ( + !(await recordStartedHandle(context, event)) || + !(await applyRewindAtStart(context, event.sessionId, current, rewind)) + ) { + return refusedStart() + } + if ( + !context.runtimeState.startupAttempts.onClock(event.sessionId, child) || + !markProviderChildStarted(current, child) + ) { + return 'stale' + } + await settleUnheldStructuredAgentSessionTurns( + current.journal, + event.sessionId, + child.fence, + event.acquisitionGeneration, + context.deps.adapter.liveTurnId?.(event.sessionId) + ).catch((error: unknown) => + context.deps.logger.warn('settling turns the agent is not running failed', { + scope: 'unheld-turns', + sessionId: event.sessionId, + error + }) + ) + const sourceChild = current.child + context.runtimeState.startupAttempts.ready(event.sessionId, child) + noteStructuredAgentSessionRuntimePermission(current.child, event.reportedOptions, true) + context.publishStatus?.(event.sessionId) + await noteStructuredAgentSessionReportedOptions( context, event.sessionId, - event.catalogListing, + child.fence, + event.reportedOptions + ) + void context.wakeDelivery(event.sessionId) + noteStructuredAgentSessionProviderStarted( + context.deps, + event.sessionId, () => context.sessions.get(event.sessionId)?.child === sourceChild && sourceChild?.catalogSource?.() !== false ) - } - context.publishStatus?.(event.sessionId) - return undefined - }) -} - -/** Reconcile uncertain rewind before delivery; persistent uncertainty fails startup. */ -async function readRewindAtStart( - context: StructuredAgentSessionProviderStartedContext, - event: OpenedProvider, - signal?: AbortSignal -): Promise { - const { sessionId } = event - const journal = context.sessions.get(sessionId)?.journal - if (!journal) { - return null - } - try { - return await readStructuredRewindRecovery( - context.deps, - sessionId, - journal, - structuredAgentSessionConversationFence(context.deps.store, sessionId), - context.deps.adapter, - signal, - recordWithStartedHandle(context, event) - ) - } catch (error) { - return rewindRecoveryFailed(context, sessionId, error, signal) - } -} - -/** Predict the committed handle for rewind cuts; if recording is refused, retain the existing record. */ -function recordWithStartedHandle( - context: StructuredAgentSessionProviderStartedContext, - { sessionId, fence, link }: OpenedProvider -): AgentSessionRecord | null { - const record = context.deps.store.getRecord(sessionId) - try { - return record && link - ? recordAgentSessionProviderHandle({ record, fence, link, now: context.now() }) - : record - } catch { - return record - } -} - -async function applyRewindAtStart( - context: StructuredAgentSessionProviderStartedContext, - sessionId: string, - session: StructuredAgentSessionHostSession, - evidence: StructuredRewindRecoveryEvidence | 'failed' -): Promise { - if (evidence === 'failed') { - return false - } - try { - await applyStructuredRewindRecovery( - context.deps, - sessionId, - session.journal, - structuredAgentSessionConversationFence(context.deps.store, sessionId), - evidence - ) - return true - } catch (error) { - rewindRecoveryFailed(context, sessionId, error) - return false - } -} - -function rewindRecoveryFailed( - context: StructuredAgentSessionProviderStartedContext, - sessionId: string, - error: unknown, - signal?: AbortSignal -): 'failed' { - if (!signal?.aborted) { - context.deps.logger.warn('settling a rewind in doubt at the start failed', { - scope: 'rewind-recovery', - sessionId, - error - }) - } - return 'failed' -} - -/** Persist the opened handle before delivery so future starts can resume; false if no handle is proven. */ -async function recordStartedHandle( - context: StructuredAgentSessionProviderStartedContext, - event: OpenedProvider -): Promise { - const { store, logger } = context.deps - const { link } = event - if (!link) { - // Only an already-proven handle permits open without a new link. - return store.getRecord(event.sessionId)?.lease.provenHandleLinkId != null - } - try { - const committed = await store.transitionHandoff(event.sessionId, (record) => - recordAgentSessionProviderHandle({ record, fence: event.fence, link, now: context.now() }) - ) - // Inherit only a committed new context; a resumed CLI keeps its own later title. - if (event.newConversation ?? (link.origin === 'created' || link.origin === 'forked')) { - context.deps.nameProviderConversation?.(committed) - } - return true - } catch (error) { - logger.warn('recording the handle a started provider answered with failed', { - scope: 'provider-started-handle', - sessionId: event.sessionId, - error + if (event.catalogListing) { + saveStructuredAgentSessionListing( + context, + event.sessionId, + event.catalogListing, + () => + context.sessions.get(event.sessionId)?.child === sourceChild && + sourceChild?.catalogSource?.() !== false + ) + } + context.publishStatus?.(event.sessionId) + return 'ready' as const }) - return false - } + ) } /** Prompt catalog recheck after startup may prove installation or restored sign-in; the probe decides. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts index 34d3d4117f1c..083758d6487d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts @@ -355,7 +355,8 @@ describe('host rewind', () => { expect( await attachAndStart(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) ).toMatchObject({ ok: true }) - expect(startedChild()).toBeNull() + // The start's owner stops the child once its required rewind read failed. + await vi.waitFor(() => expect(startedChild()).toBeNull()) expect(store.getRecord(HOST_TEST_SESSION)?.rewind?.phase).toBe('prepared') expect( await attachAndStart(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts index 802a46f2ee43..060cb79332db 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts @@ -1,5 +1,3 @@ -import { retryableStructuredAgentSessionStart } from './structured-agent-session-start-retry' -import { withTimeout } from '../../../shared/promise-timeout-fallback' import { continuingStructuredAgentSessionInput } from './structured-agent-session-handing-over' import { randomUUID } from 'node:crypto' import { setTimeout as delay } from 'node:timers/promises' @@ -7,30 +5,28 @@ import { throwIfSignalAborted, waitForPromiseWithSignal } from '../../../shared/ import { refuseUnclassified } from '../../../shared/agent-session/failures/agent-session-wire-refusals' import { agentSessionWriteNoticeEnglish } from '../../../shared/agent-session/failures/agent-session-refusal-notice' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session/requests/agent-session-queued-submission' -import { - agentSessionFailureFact, - type SubmissionRejectionFact -} from '../../../shared/agent-session/failures/agent-session-failure' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionAttachLock } from './structured-agent-session-attach-lock' import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-agent-start' -import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import type { + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity +} from './structured-agent-session-host-types' import { structuredAgentSessionStartFlightLock } from './structured-agent-session-start-flight' -import { isStructuredAgentSessionStartupExpired } from './structured-agent-session-startup-attempt-contract' +import { + isStructuredAgentSessionStartupContractBreach, + isStructuredAgentSessionStartupExpired +} from './structured-agent-session-startup-attempt-contract' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' import { refusedStructuredAgentSessionStart } from './structured-agent-session-start-eligibility' import { refusedStructuredAgentSessionStart as failedStart, recordStructuredAgentSessionStartFailure } from './structured-agent-session-start-failure-row' -import { - providerStartupFailureFact, - withObservedProviderExit, - structuredAgentSessionStartFailure, - structuredAgentSessionStartFailureFact -} from './structured-agent-session-failure-text' +import { structuredAgentSessionStartFailure } from './structured-agent-session-failure-text' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' import { sameProviderChild } from './structured-agent-session-provider-child' -import { providerRequestTimeoutOf } from '../../provider-process/provider-request-timeout' export function registerStructuredAgentSessionStart( context: StructuredAgentSessionAttachContext, @@ -80,87 +76,16 @@ export function registerStructuredAgentSessionStart( } result = await run(lock, attemptId) if (result.ok) { + // The child's open owns its outcome; readiness ends once that open gave it. const ready = await lock.wait(handle.ended) throwIfSignalAborted(acquire.signal) if (ready === 'ready') { return result } - const opening = handle.result.opening - const join = - opening !== undefined && - handle.result.retry !== false && - !finalStartupExitFailure(context.sessions.get(sessionId)?.lastEndedChild?.failure) - if (join) { - // The ended child disarmed the ceiling timer; the join must not outlast it. - await lock.wait( - waitForPromiseWithSignal( - withTimeout( - opening, - Math.max(0, Math.min(2_000, attempts.untilCeiling(registeredAt))), - undefined - ), - acquire.signal - ) - ) - } - // The flight owns this open after exit, whether or not its answer was awaited. - handle.result.cancelOpen?.() - throwIfSignalAborted(acquire.signal) - if (join && attempts.pastCeiling(registeredAt)) { - result = { ...failedStructuredAgentSessionStart(), expired: true } - break - } - result = await lock.commit(async () => { - const session = context.sessions.get(sessionId) - const ended = session?.lastEndedChild - const child = handle.result.child - if ( - !child || - !ended || - !sameProviderChild(ended, child) || - !ended.rootGone || - (session?.child && sameProviderChild(session.child, child)) - ) { - return refusedStructuredAgentSessionStart( - 'agent_session_ownership_unknown', - { reason: 'ownerUnproven' }, - 'The previous agent process could not be verified as stopped.' - ) - } - const finalExit = finalStartupExitFailure(ended.failure) - const openError = handle.result.openError - // An elapsed provider deadline keeps its words; the later exit doesn't explain it. - const openFailure = - ended.cause === 'exit' && - openError instanceof Error && - !providerRequestTimeoutOf(openError) - ? providerStartupFailureFact(withObservedProviderExit(openError)) - : handle.result.failure - const failure = structuredAgentSessionStartFailureFact({ - exit: finalExit - ? ended.failure - : openFailure - ? { ...openFailure, detail: openFailure.detail ?? ended.failure?.detail } - : ended.failure - }) - return { - ...failedStructuredAgentSessionStart(), - startKey: ended.generation, - failure, - retry: - !finalExit && - ('openError' in handle.result - ? retryableStructuredAgentSessionStart( - handle.result.openError, - ended.cause === 'exit' - ) - : (handle.result.retry ?? - (ended.cause === 'exit' && - failure.kind === 'providerStartFailed' && - !failure.refusal && - !failure.requestTimeout))) - } - }) + const owned = handle.result + result = await lock.commit(async () => + structuredAgentSessionOpenFailure(context.sessions.get(sessionId), owned) + ) } if (result.ok || result.aborted || result.expired || !result.retry || number === 4) { break @@ -194,12 +119,7 @@ export function registerStructuredAgentSessionStart( } return context.serialize(sessionId, async () => { const settled = acquire.signal.aborted - ? { - ...failedStructuredAgentSessionStart(), - ...(isStructuredAgentSessionStartupExpired(acquire.signal.reason) - ? { expired: true as const } - : { aborted: true as const }) - } + ? stoppedStructuredAgentSessionStart(acquire.signal.reason) : result if (!settled.aborted && (startedFor !== undefined || children.length > 0)) { const session = context.sessions.get(sessionId) @@ -264,11 +184,51 @@ export function structuredAgentSessionStartStillQueued( ) } -function finalStartupExitFailure(failure: SubmissionRejectionFact | undefined): boolean { - return ( - !!failure && - (!['providerExited', 'providerStartFailed', 'startFailed'].includes(failure.kind) || - !!failure.refusal || - !!failure.requestTimeout) - ) +/** A safe verdict replays only once this child's exit is proven; no verdict replays nothing. */ +function structuredAgentSessionOpenFailure( + session: Pick | undefined, + { + child, + failure + }: { + child: StructuredAgentSessionProviderChildIdentity | null + failure?: AgentSessionStartFailure + } +): StructuredAgentSessionResumeOutcome { + const ended = session?.lastEndedChild + const exitProven = + !!child && + !!ended?.rootGone && + sameProviderChild(ended, child) && + !(session?.child && sameProviderChild(session.child, child)) + if (!child || !failure || (failure.retry === 'safe' && !exitProven)) { + return refusedStructuredAgentSessionStart( + 'agent_session_ownership_unknown', + { reason: 'ownerUnproven' }, + 'The previous agent process could not be verified as stopped.' + ) + } + return { + ...failedStructuredAgentSessionStart(), + startKey: child.generation, + failure: failure.fact, + retry: failure.retry === 'safe' + } +} + +/** The deadline keeps its own outcome; a broken adapter contract fails the start; anything else + * stopping it is cancellation. */ +function stoppedStructuredAgentSessionStart( + reason: unknown +): Extract { + if (isStructuredAgentSessionStartupExpired(reason)) { + return { ...failedStructuredAgentSessionStart(), expired: true } + } + if (isStructuredAgentSessionStartupContractBreach(reason)) { + return { + ...failedStructuredAgentSessionStart(), + failure: agentSessionFailureFact('providerStartFailed') + } + } + return { ...failedStructuredAgentSessionStart(), aborted: true } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts index 6f718e6f44fe..eec28f5b17b6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts @@ -8,6 +8,7 @@ import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session/a import type { AgentSessionRecordStore } from '../../runtime/agent-session/records/agent-session-record-store' import { openTestAgentSessionRecordStore } from '../../runtime/agent-session/records/agent-session-record-store-test-harness' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { structuredAgentSessionHeldStarts } from './structured-agent-session-held-start.test-support' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { HOST_TEST_NOW as NOW, @@ -46,6 +47,7 @@ let generation = 0 let exitOnStart = false let dispatch: Mock let frames: AgentSessionSubscribeEvent[] = [] +const starts = structuredAgentSessionHeldStarts(() => host) function exitBeforeProof(): Promise { return host.handleAdapterEvent({ @@ -105,6 +107,7 @@ beforeEach(async () => { resetHostTestOperationIds() generation = 0 frames = [] + starts.clear() exitOnStart = false dispatch = vi.fn(async () => ({ state: 'admitted' as const })) store = await openTestAgentSessionRecordStore(root) @@ -112,9 +115,10 @@ beforeEach(async () => { agents: NO_STRUCTURED_AGENTS, logger: createStructuredAgentSessionLogger(), store, - adapter: { + // The adapter answers its never-proved opens with its own child's exit. + adapter: starts.wrap({ open: () => new Promise(() => {}), - acquire: vi.fn(async ({ fence, spawnToken }) => { + acquire: starts.held(async ({ fence, spawnToken }) => { const child = { process: { hostId: 'local', @@ -150,7 +154,7 @@ beforeEach(async () => { cancelTurn: vi.fn(async () => ({ cancelled: true })), answerPrompt: vi.fn(async () => undefined), setOption: vi.fn(async () => undefined) - }, + }), journalDatabase: openTestJournalHostDatabase(root), claimKeyId: 'key-1', mintSpawnToken: () => `spawn-${generation + 1}`, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts new file mode 100644 index 000000000000..8e0708440882 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts @@ -0,0 +1,205 @@ +// The adapter that holds the protocol connection decides whether a failed start may run again; +// the host reads only this verdict, never provider error classes. + +import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { findInErrorChain } from '../../provider-process/provider-error-chain' +import { + providerExecutableMissing, + providerLaunchFolderMissing +} from '../../provider-process/provider-executable-missing' +import { providerRequestTimeoutOf } from '../../provider-process/provider-request-timeout' +import { argumentProblemOf } from '../structured-agent-arguments-error' +import { agentCliInstallationProblemOf } from '../agent-cli-installation-error' +import { + AgentSessionAcquisitionRefusal, + isAgentSessionPreSpawnError +} from './structured-agent-session-adapter-errors' +import { + AgentSessionAcquisitionExitProvenError, + AgentSessionAcquisitionExitUnprovenError, + AgentSessionAcquisitionRootExitObservedError +} from './structured-agent-session-acquisition-errors' +import { + providerExitObserved, + providerStartupFailureFact, + withObservedProviderExit +} from './structured-agent-session-failure-text' + +/** `safe`: the agent gave no answer before its output ended, or declared its answer temporary. */ +export type AgentSessionStartRetry = 'safe' | 'never' + +/** An adapter's reading of one error in the chain: the agent's answer, final or declared + * temporary, or its own transport ending with no answer once output drained. */ +export type AgentSessionStartAnswer = (cause: Error) => 'final' | 'temporary' | 'exited' | undefined + +export class AgentSessionStartFailure extends Error { + readonly fact: SubmissionRejectionFact + + constructor( + cause: unknown, + readonly retry: AgentSessionStartRetry, + fact?: SubmissionRejectionFact + ) { + super(cause instanceof Error ? cause.message : String(cause), { cause }) + this.name = 'AgentSessionStartFailure' + this.fact = fact ?? providerStartupFailureFact(cause) + } +} + +/** `exited`: the adapter saw its process end on its own, which words even a final answer as the + * agent's own failure to start without changing whether it may run again. */ +export function agentSessionStartFailure( + error: unknown, + answer?: AgentSessionStartAnswer, + exited = false +): AgentSessionStartFailure { + if (error instanceof AgentSessionStartFailure) { + return error + } + const disposition = startDisposition(error, answer) + // An adapter's own exit error words the start as the agent's failure, as an observed exit does. + const worded = + (exited || disposition.exited) && error instanceof Error + ? withObservedProviderExit(new Error(error.message, { cause: error })) + : error + return new AgentSessionStartFailure(error, disposition.retry, providerStartupFailureFact(worded)) +} + +/** The adapter's verdict on an error where it read an answer or its own exit; otherwise none. */ +export function agentSessionStartVerdict( + error: unknown, + answer: AgentSessionStartAnswer +): AgentSessionStartFailure | undefined { + if (error instanceof AgentSessionStartFailure) { + return error + } + const answered = findInErrorChain(error, (cause) => + answer(cause) === undefined ? undefined : true + ) + return answered ? agentSessionStartFailure(error, answer) : undefined +} + +/** Acquisition failures keep at their top the exit proofs and refusals lease bookkeeping reads; + * the adapter's verdict goes beneath them, so a final answer outlives its cleanup's proof. */ +export function agentSessionAcquireFailure( + error: unknown, + answer: AgentSessionStartAnswer +): unknown { + if ( + error instanceof AgentSessionAcquisitionRootExitObservedError || + error instanceof AgentSessionAcquisitionExitProvenError + ) { + const verdict = agentSessionStartVerdict(error.cause, answer) + if (!verdict || verdict === error.cause) { + return error + } + return error instanceof AgentSessionAcquisitionRootExitObservedError + ? new AgentSessionAcquisitionRootExitObservedError(verdict, error.deathEvidenceKind) + : new AgentSessionAcquisitionExitProvenError(verdict) + } + if ( + error instanceof AgentSessionAcquisitionExitUnprovenError || + error instanceof AgentSessionAcquisitionRefusal || + isAgentSessionPreSpawnError(error) + ) { + return error + } + return agentSessionStartVerdict(error, answer) ?? error +} + +/** Each session's latest acquisition that ended before the host committed it ready, with the + * adapter's verdict; the session's next acquisition or the adapter's close disposes it. */ +export class AgentSessionStartEndings { + private readonly endings = new Map< + string, + { generation: string; failure: AgentSessionStartFailure } + >() + + /** The first ending of an acquisition is its verdict; later reports of it change nothing. */ + record(sessionId: string, generation: string, failure: AgentSessionStartFailure): void { + if (this.endings.get(sessionId)?.generation !== generation) { + this.endings.set(sessionId, { generation, failure }) + } + } + + get(sessionId: string, generation: string): AgentSessionStartFailure | undefined { + const ended = this.endings.get(sessionId) + return ended?.generation === generation ? ended.failure : undefined + } + + forget(sessionId: string): void { + this.endings.delete(sessionId) + } + + clear(): void { + this.endings.clear() + } +} + +/** The verdict for an error no adapter converted, such as an acquisition the host wrapped. */ +export function agentSessionStartRetry(error: unknown): AgentSessionStartRetry { + return error instanceof AgentSessionStartFailure ? error.retry : startDisposition(error).retry +} + +/** A failed acquisition replays only once its exit is proven and the adapter's verdict allows it. */ +export function retryableAgentSessionAcquisition(error: unknown): boolean { + return ( + (error instanceof AgentSessionAcquisitionExitProvenError || + error instanceof AgentSessionAcquisitionRootExitObservedError) && + agentSessionStartRetry(error) === 'safe' + ) +} + +/** A final answer anywhere in the chain wins. Cleanup's exit proof alone says nothing about the + * agent's answer, so only the agent's own exit, a declared temporary answer or a launch short of + * host resources permits one more run. */ +function startDisposition( + error: unknown, + answer?: AgentSessionStartAnswer +): { retry: AgentSessionStartRetry; exited: boolean } { + if ( + providerRequestTimeoutOf(error) || + argumentProblemOf(error) || + agentCliInstallationProblemOf(error) || + providerExecutableMissing(error) || + providerLaunchFolderMissing(error) || + findInErrorChain(error, (cause) => + cause instanceof AggregateError || + cause instanceof AgentSessionAcquisitionExitUnprovenError || + cause instanceof AgentSessionAcquisitionRefusal || + isAgentSessionPreSpawnError(cause) + ? true + : undefined + ) + ) { + return { retry: 'never', exited: false } + } + const answers = new Set | AgentSessionStartRetry>() + findInErrorChain(error, (cause) => { + answers.add(cause instanceof AgentSessionStartFailure ? cause.retry : answer?.(cause)) + // An adapter's verdict already covers the errors beneath it. + return cause instanceof AgentSessionStartFailure ? true : undefined + }) + if (answers.has('never') || answers.has('final')) { + return { retry: 'never', exited: false } + } + const exited = answers.has('exited') + const safe = + exited || + answers.has('safe') || + answers.has('temporary') || + providerExitObserved(error) || + providerResourcePressure(error) + return { retry: safe ? 'safe' : 'never', exited } +} + +/** The host ran short of processes or files; the same launch can run once that passes. */ +export function providerResourcePressure(error: unknown): boolean { + return ( + findInErrorChain(error, (cause) => + 'code' in cause && ['EAGAIN', 'EMFILE', 'ENFILE'].some((code) => code === cause.code) + ? true + : undefined + ) === true + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-provider-answer.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-provider-answer.ts new file mode 100644 index 000000000000..d62e6f736bd2 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-provider-answer.ts @@ -0,0 +1,142 @@ +import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { recordAgentSessionProviderHandle } from '../../runtime/agent-session/ownership/agent-session-provider-handle-transition' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import type { StructuredAgentSessionProviderOpenResult } from './structured-agent-session-provider-open-contract' +import type { StructuredAgentSessionProviderStartedContext } from './structured-agent-session-provider-started' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' +import { + AgentSessionStartFailure, + agentSessionStartFailure +} from './structured-agent-session-start-failure' +import { + applyStructuredRewindRecovery, + readStructuredRewindRecovery, + type StructuredRewindRecoveryEvidence +} from './structured-rewind-recovery' + +// What a starting child's open answered, read and recorded before the start is ready. + +export type OpenedProvider = StructuredAgentSessionProviderOpenResult & { + sessionId: string + fence: number + acquisitionGeneration: string +} + +/** Orca could not record or apply what the open answered; nothing about the agent says retry. */ +export function refusedStart(): AgentSessionStartFailure { + return new AgentSessionStartFailure( + new Error('Orca could not record what the agent answered at its start'), + 'never', + agentSessionFailureFact('hostStopped') + ) +} + +/** Reconcile uncertain rewind before delivery; persistent uncertainty fails startup with the + * adapter's verdict on its read, or as final when Orca's own reading failed. */ +export async function readRewindAtStart( + context: StructuredAgentSessionProviderStartedContext, + event: OpenedProvider, + signal?: AbortSignal +): Promise { + const { sessionId } = event + const journal = context.sessions.get(sessionId)?.journal + if (!journal) { + return null + } + try { + return await readStructuredRewindRecovery( + context.deps, + sessionId, + journal, + structuredAgentSessionConversationFence(context.deps.store, sessionId), + context.deps.adapter, + signal, + recordWithStartedHandle(context, event) + ) + } catch (error) { + rewindRecoveryFailed(context, sessionId, error, signal) + return agentSessionStartFailure(error) + } +} + +/** Predict the committed handle for rewind cuts; if recording is refused, retain the existing record. */ +function recordWithStartedHandle( + context: StructuredAgentSessionProviderStartedContext, + { sessionId, fence, link }: OpenedProvider +): AgentSessionRecord | null { + const record = context.deps.store.getRecord(sessionId) + try { + return record && link + ? recordAgentSessionProviderHandle({ record, fence, link, now: context.now() }) + : record + } catch { + return record + } +} + +export async function applyRewindAtStart( + context: StructuredAgentSessionProviderStartedContext, + sessionId: string, + session: StructuredAgentSessionHostSession, + evidence: StructuredRewindRecoveryEvidence +): Promise { + try { + await applyStructuredRewindRecovery( + context.deps, + sessionId, + session.journal, + structuredAgentSessionConversationFence(context.deps.store, sessionId), + evidence + ) + return true + } catch (error) { + rewindRecoveryFailed(context, sessionId, error) + return false + } +} + +function rewindRecoveryFailed( + context: StructuredAgentSessionProviderStartedContext, + sessionId: string, + error: unknown, + signal?: AbortSignal +): void { + if (!signal?.aborted) { + context.deps.logger.warn('settling a rewind in doubt at the start failed', { + scope: 'rewind-recovery', + sessionId, + error + }) + } +} + +/** Persist the opened handle before delivery so future starts can resume; false if no handle is proven. */ +export async function recordStartedHandle( + context: StructuredAgentSessionProviderStartedContext, + event: OpenedProvider +): Promise { + const { store, logger } = context.deps + const { link } = event + if (!link) { + // Only an already-proven handle permits open without a new link. + return store.getRecord(event.sessionId)?.lease.provenHandleLinkId != null + } + try { + const committed = await store.transitionHandoff(event.sessionId, (record) => + recordAgentSessionProviderHandle({ record, fence: event.fence, link, now: context.now() }) + ) + // Inherit only a committed new context; a resumed CLI keeps its own later title. + if (event.newConversation ?? (link.origin === 'created' || link.origin === 'forked')) { + context.deps.nameProviderConversation?.(committed) + } + return true + } catch (error) { + logger.warn('recording the handle a started provider answered with failed', { + scope: 'provider-started-handle', + sessionId: event.sessionId, + error + }) + return false + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts index 555d8c51b6b0..86aa14d499d7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts @@ -22,11 +22,31 @@ import { import { AgentSessionAcquisitionRefusal } from './structured-agent-session-adapter-errors' import { withObservedProviderExit } from './structured-agent-session-failure-text' import { - retryableStructuredAgentSessionAcquisition, - retryableStructuredAgentSessionStart -} from './structured-agent-session-start-retry' + agentSessionAcquireFailure, + agentSessionStartFailure, + retryableAgentSessionAcquisition, + type AgentSessionStartAnswer +} from './structured-agent-session-start-failure' +import { codexStartupAnswer } from '../../codex/app-server/codex-startup-outcome' +import { acpStartupAnswer } from '../../acp/acp-startup-outcome' +import { piStartupAnswer } from '../../pi/rpc-startup-outcome' +import { claudeStartupAnswer } from '../../claude/claude-startup-failure' import captures from './__fixtures__/startup-protocol-rejections.json' +// Each adapter reads only its own answers; the shared rule then gives the generic verdict. +const adapterAnswer: AgentSessionStartAnswer = (cause) => + [codexStartupAnswer, acpStartupAnswer, piStartupAnswer, claudeStartupAnswer] + .map((answer) => answer(cause)) + .find((verdict) => verdict !== undefined) + +function retryableStructuredAgentSessionStart(error: unknown): boolean { + return agentSessionStartFailure(error, adapterAnswer).retry === 'safe' +} + +function retryableStructuredAgentSessionAcquisition(error: unknown): boolean { + return retryableAgentSessionAcquisition(agentSessionAcquireFailure(error, adapterAnswer)) +} + function expectExitEvidenceRequired(error: unknown) { expect(retryableStructuredAgentSessionStart(error)).toBe(true) expect(retryableStructuredAgentSessionAcquisition(error)).toBe(false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts deleted file mode 100644 index 63b035e1175a..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' -import { claudeStartupAnswer } from '../../claude/claude-startup-failure' -import { temporaryCodexStartupAnswer } from '../../codex/app-server/codex-startup-failure' -import { CodexAppServerUnsupportedError } from '../../codex/app-server/codex-app-server-session' -import { temporaryAcpStartupAnswer } from '../../acp/acp-startup-failure' -import { AcpRpcError } from '../../acp/acp-errors' -import { JsonlRpcResponseError } from '../../jsonl-rpc/peer' -import { findInErrorChain } from '../../provider-process/provider-error-chain' -import { - providerExecutableMissing, - providerLaunchFolderMissing -} from '../../provider-process/provider-executable-missing' -import { providerRequestTimeoutOf } from '../../provider-process/provider-request-timeout' -import { argumentProblemOf } from '../structured-agent-arguments-error' -import { agentCliInstallationProblemOf } from '../agent-cli-installation-error' -import { - AgentSessionAcquisitionExitProvenError, - AgentSessionAcquisitionRootExitObservedError, - AgentSessionAcquisitionExitUnprovenError, - AgentSessionAcquisitionRefusal, - isAgentSessionPreSpawnError -} from './structured-agent-session-adapter' -import { providerExitObserved } from './structured-agent-session-failure-text' - -function agentStartAnswer(error: Error): 'temporary' | 'final' | undefined { - if (error instanceof CodexAppServerRequestError) { - return temporaryCodexStartupAnswer(error) ? 'temporary' : 'final' - } - if (error instanceof CodexAppServerUnsupportedError || error instanceof JsonlRpcResponseError) { - return 'final' - } - if (error instanceof AcpRpcError) { - return temporaryAcpStartupAnswer(error) ? 'temporary' : 'final' - } - return claudeStartupAnswer(error) -} - -/** Only positive provider evidence retries; an Orca fault or a typed refusal stays final. */ -export function retryableStructuredAgentSessionStart( - error: unknown, - unexpectedExit = false -): boolean { - if ( - providerRequestTimeoutOf(error) || - argumentProblemOf(error) || - agentCliInstallationProblemOf(error) || - providerExecutableMissing(error) || - providerLaunchFolderMissing(error) - ) { - return false - } - const final = findInErrorChain(error, (cause) => - cause instanceof AggregateError || - cause instanceof AgentSessionAcquisitionExitUnprovenError || - cause instanceof AgentSessionAcquisitionRefusal || - isAgentSessionPreSpawnError(cause) || - agentStartAnswer(cause) === 'final' - ? true - : undefined - ) - if (final) { - return false - } - return ( - unexpectedExit || - providerExitObserved(error) || - findInErrorChain(error, (cause) => - agentStartAnswer(cause) === 'temporary' || - ('code' in cause && ['EAGAIN', 'EMFILE', 'ENFILE'].some((code) => code === cause.code)) - ? true - : undefined - ) === true - ) -} - -export function retryableStructuredAgentSessionAcquisition(error: unknown): boolean { - return ( - (error instanceof AgentSessionAcquisitionExitProvenError || - error instanceof AgentSessionAcquisitionRootExitObservedError) && - retryableStructuredAgentSessionStart(error) - ) -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts index 9de29b0b463a..7901fe7326a9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as TimerPromises from 'node:timers/promises' import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' import { withProviderRequestTimeout } from '../../provider-process/provider-request-timeout' import { AgentSessionAcquisitionRefusal } from './structured-agent-session-adapter' import { agentSessionFailureWords } from '../../../shared/agent-session/failures/agent-session-failure-words' @@ -8,6 +9,8 @@ import { agentSessionFailureFact } from '../../../shared/agent-session/failures/ import { DISPATCH_REJECTED_CANCELLED } from '../../../shared/agent-session/requests/structured-agent-session-dispatch-rejection' import * as startFlight from './structured-agent-session-start-flight' import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from './structured-agent-session-send-settlement' +import { agentSessionStartFailure } from './structured-agent-session-start-failure' +import { withObservedProviderExit } from './structured-agent-session-failure-text' import { CALLER, attach, @@ -119,7 +122,12 @@ async function endAttempt() { reason: 'exited while starting', cause: 'unexpected-exit' }) - openFailures.get(child.generation)?.(new Error('provider exited before open answered')) + // As the adapter: its open answers with this acquisition's own unanswered exit. + openFailures.get(child.generation)?.( + agentSessionStartFailure( + withObservedProviderExit(new Error('provider exited before open answered')) + ) + ) openFailures.delete(child.generation) await eventually(() => expect(session.child).toBeNull()) } @@ -170,10 +178,15 @@ describe('final logical startup settlement', () => { await lifetime.failStartingChild( SESSION, sessions.get(SESSION)!.child!, - withProviderRequestTimeout(new CodexAppServerRequestError('initialize', -1, 'deadline'), { - method: 'initialize', - timeoutMs: 60_000 - }), + codexStartFailure( + withProviderRequestTimeout( + new CodexAppServerRequestError('initialize', -1, 'deadline'), + { + method: 'initialize', + timeoutMs: 60_000 + } + ) + ), 60_000 ) await expect(flight.outcome).resolves.toMatchObject({ ok: false, aborted: true }) @@ -263,10 +276,12 @@ describe('final logical startup settlement', () => { await lifetime.failStartingChild( SESSION, sessions.get(SESSION)!.child!, - withProviderRequestTimeout(new CodexAppServerRequestError('initialize', -1, 'deadline'), { - method: 'initialize', - timeoutMs: 60_000 - }), + codexStartFailure( + withProviderRequestTimeout(new CodexAppServerRequestError('initialize', -1, 'deadline'), { + method: 'initialize', + timeoutMs: 60_000 + }) + ), 60_000 ) await eventually(() => expect(conversationDelivery.loop.isRunning(SESSION)).toBe(false)) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-contract.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-contract.ts index 5ae27374e38b..962060db4402 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-contract.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-contract.ts @@ -49,6 +49,22 @@ export function isStructuredAgentSessionStartupExpired(reason: unknown): boolean return reason instanceof Error && reason.name === 'StructuredAgentSessionStartupExpiredError' } +/** How long after an acquisition's own root exit its adapter may take to answer for the start. + * Every adapter's post-exit work (output drain, initialize-answer join) finishes well inside it. */ +export const STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS = 5_000 + +/** The adapter gave no outcome within the grace after its process exited: a broken contract. */ +export class StructuredAgentSessionStartupContractError extends Error { + constructor(agent: string) { + super(`the ${agent} adapter gave no startup outcome after its process exited`) + this.name = 'StructuredAgentSessionStartupContractError' + } +} + +export function isStructuredAgentSessionStartupContractBreach(reason: unknown): boolean { + return reason instanceof Error && reason.name === 'StructuredAgentSessionStartupContractError' +} + export type StructuredAgentSessionAcquisitionReceipt = Pick< StructuredAgentSessionStartupAttempt, 'attemptId' | 'fence' | 'spawnToken' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts index 93ed23dd44d8..597a1eb4972b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts @@ -33,6 +33,8 @@ import { import { StructuredAgentSessionHost } from './structured-agent-session-host' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import type { StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import { testAdapterStartupEnding } from './structured-agent-session-held-start.test-support' export const CALLER = { callerKey: 'client-1' } @@ -50,6 +52,8 @@ export const providerOpen = vi.fn void>() +const endings = new Map() + const openGates = new Map void>() export async function proveOpen(model = 'default'): Promise { @@ -59,6 +63,23 @@ export async function proveOpen(model = 'default'): Promise { openGates.delete(generation) } +/** As an adapter whose open rejects with its own verdict for the current acquisition. */ +export function failOpen(error: Error): void { + const generation = `generation-${acquire.mock.calls.length}` + openFailures.get(generation)?.(error) + openFailures.delete(generation) +} + +/** As an adapter seeing the current acquisition's own root exit. */ +export function observeRootExit(): void { + acquire.mock.calls.at(-1)?.[0].events?.observeRootExit?.() +} + +/** As an adapter recording the current acquisition's unanswered death before it was ready. */ +export function recordStartupEnding(): void { + endings.set(`generation-${acquire.mock.calls.length}`, testAdapterStartupEnding()) +} + export function acquisition(input: StructuredAgentSessionAcquireInput): AgentSessionAcquisition { return { process: { @@ -97,6 +118,7 @@ async function startHost({ startupLimits, supportsGoals }: StartupHostOptions = dispatch, closeSession, releaseAcquisition: vi.fn(async () => true), + startupEnding: (_sessionId, generation) => endings.get(generation), cancelTurn: vi.fn(async () => ({ cancelled: false })), answerPrompt: vi.fn(async () => undefined), setOption @@ -125,10 +147,18 @@ async function startHost({ startupLimits, supportsGoals }: StartupHostOptions = }) const handleEvent = host.handleAdapterEvent.bind(host) host.handleAdapterEvent = async (event) => { + // As an adapter: its own ending of this exact acquisition answers the open and the start. + const ending = + event.type === 'ended' && event.cause === 'unexpected-exit' && event.acquisitionGeneration + ? testAdapterStartupEnding(event.failure) + : undefined + if (ending && event.type === 'ended') { + endings.set(event.acquisitionGeneration, ending) + } const result = await handleEvent(event) if (event.type === 'ended' && event.acquisitionGeneration) { openFailures.get(event.acquisitionGeneration)?.( - new Error('provider exited before open answered') + ending ?? new Error('provider exited before open answered') ) openFailures.delete(event.acquisitionGeneration) } @@ -160,6 +190,7 @@ export function registerStartupAttemptHostTests(): void { recordLiveListing.mockReset() openFailures.clear() openGates.clear() + endings.clear() providerOpen.mockReset().mockImplementation( ({ acquisitionGeneration }) => new Promise((resolve, reject) => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts index 4e216c1c68ed..54e3e0dff6c8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' import { withProviderRequestTimeout } from '../../provider-process/provider-request-timeout' import { AgentSessionAcquisitionRefusal, @@ -15,14 +16,12 @@ import { projectStructuredAgentSessionMessages } from '../../../shared/agent-ses import { HOST_TEST_SESSION as SESSION, HOST_TEST_THREAD as THREAD, - hostTestAttachParams, hostTestOperationId } from './structured-agent-session-host-test-data' -import type { StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' import { - providerStartupFailureFact, - withObservedProviderExit -} from './structured-agent-session-failure-text' + STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS, + type StructuredAgentSessionStartupLimits +} from './structured-agent-session-startup-attempt-contract' import { CALLER, accept, @@ -41,7 +40,10 @@ import { stop, store, submission, - registerStartupAttemptHostTests + registerStartupAttemptHostTests, + failOpen, + observeRootExit, + recordStartupEnding } from './structured-agent-session-startup-attempt-host-test-support' let startupLimits: Partial | undefined @@ -357,55 +359,39 @@ describe('one logical start across physical attempts', () => { } ) - it.each([ - agentSessionFailureFact('fullAccessAsRoot'), - agentSessionFailureFact('historyTooLarge'), - // A refusal whose reason has no failure kind of its own keeps only its refusal marker. - providerStartupFailureFact( - withObservedProviderExit(new AgentSessionAcquisitionRefusal('typed startup refusal')) - ) - ])( - 'keeps the typed $kind exit-only refusal final even when raw open never settles', - async (failure) => { - await restartWith(publishFirst) - let signal: AbortSignal | undefined - providerOpen.mockImplementationOnce((input) => { - signal = input.signal - return new Promise(() => {}) - }) - const id = await heldBehindStart('one request') - const child = host.collaboratorsForTests().sessions.get(SESSION)!.child! - await host.handleAdapterEvent({ - type: 'ended', - sessionId: SESSION, - fence: child.fence, - acquisitionGeneration: child.generation!, - reason: 'root refusal', - cause: 'unexpected-exit', - failure - }) - await vi.advanceTimersByTimeAsync(2_100) - await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) - expect((await submission(id))?.rejection).toMatchObject({ kind: failure.kind }) - expect(acquire).toHaveBeenCalledTimes(1) - expect(dispatch).not.toHaveBeenCalled() - expect(signal?.aborted).toBe(true) - } - ) - - it('retires a direct attach open, which has no start flight, when its child exits', async () => { + it('fails a start whose adapter never answers after its own exit, before the ceiling and without a replacement', async () => { await restartWith(publishFirst) let signal: AbortSignal | undefined providerOpen.mockImplementationOnce((input) => { signal = input.signal return new Promise(() => {}) }) - const fence = store.getRecord(SESSION)!.lease.runtimeFence - expect(await host.attach(CALLER, hostTestAttachParams(fence))).toMatchObject({ ok: true }) - await eventually(() => expect(signal).toBeDefined()) - expect(host.collaboratorsForTests().runtimeState.startFlights.has(SESSION)).toBe(false) - await endAttempt() + const id = await heldBehindStart('one request') + // The adapter saw its process exit, but neither its open nor its exit report ever settles. + observeRootExit() + await vi.advanceTimersByTimeAsync(STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect(await submission(id)).toMatchObject( + agentSessionFailureWords(agentSessionFailureFact('providerStartFailed'), { + agentName: 'Codex', + surface: 'rejection' + }) + ) expect(signal?.aborted).toBe(true) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('recovers when the child dies after its open answered but before the start was ready', async () => { + await restartWith(publishFirst) + const id = await heldBehindStart('one request') + // The adapter's own record of this acquisition ending lands before the host commits ready. + recordStartupEnding() + await proveOpen() + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + await proveOpen() + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(dispatch).toHaveBeenCalledTimes(1) }) it('keeps an initialize deadline that rejects after the exit it raced', async () => { @@ -421,26 +407,16 @@ describe('one logical start across physical attempts', () => { expect(acquire).toHaveBeenCalledTimes(1) }) - it('bounds an unanswered open after exit and recovers without waiting for the startup ceiling', async () => { - await restartWith(publishFirst) - providerOpen.mockImplementationOnce(() => new Promise(() => {})) - const id = await heldBehindStart('one request') - await endAttempt() - await vi.advanceTimersByTimeAsync(2_200) - await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) - await proveOpen() - await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) - expect(dispatch).toHaveBeenCalledTimes(1) - }) - it('reconnects after a declared-temporary open answer and delivers the held message once', async () => { await restartWith(publishFirst) providerOpen.mockRejectedValueOnce( - new CodexAppServerRequestError( - 'initialize', - -32001, - 'Server overloaded; retry later.', - 'Server overloaded; retry later.' + codexStartFailure( + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) ) ) const id = await heldBehindStart('recover') @@ -542,15 +518,7 @@ describe('one logical start across physical attempts', () => { closeSession.mockResolvedValue(false) const second = await heldBehindStart('unproven process') const child = host.collaboratorsForTests().sessions.get(SESSION)!.child! - const { lifetime } = host.collaboratorsForTests() - await expect( - lifetime.failStartingChild( - SESSION, - child, - new CodexAppServerRequestError('thread/start', -1, 'refused'), - 1 - ) - ).rejects.toThrow('eviction failed') + failOpen(codexStartFailure(new CodexAppServerRequestError('thread/start', -1, 'refused'))) await eventually(async () => expect((await submission(second))?.dispatchState).toBe('rejected')) expect(host.collaboratorsForTests().sessions.get(SESSION)?.child).toBe(child) expect(acquire).toHaveBeenCalledTimes(1) @@ -633,17 +601,15 @@ describe('one logical start across physical attempts', () => { it('keeps an elapsed provider-step deadline final and preserves its words', async () => { await restartWith(publishFirst) const first = await heldBehindStart('initialize deadline') - const { sessions, lifetime } = host.collaboratorsForTests() const requestTimeout = { method: 'initialize', timeoutMs: 60_000 } await vi.advanceTimersByTimeAsync(60_000) - await lifetime.failStartingChild( - SESSION, - sessions.get(SESSION)!.child!, - withProviderRequestTimeout( - new CodexAppServerRequestError('initialize', -1, 'deadline'), - requestTimeout - ), - 60_000 + failOpen( + codexStartFailure( + withProviderRequestTimeout( + new CodexAppServerRequestError('initialize', -1, 'deadline'), + requestTimeout + ) + ) ) await eventually(async () => expect((await submission(first))?.dispatchState).toBe('rejected')) expect(await submission(first)).toMatchObject( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts index cd02662b1aac..ab65901e2ec7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts @@ -1,57 +1,43 @@ -import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' import { waitForStructuredAgentSessionStart, type StructuredAgentSessionStartReadiness } from './structured-agent-session-start-readiness' export type { StructuredAgentSessionStartReadiness } from './structured-agent-session-start-readiness' -import type { AgentSessionJournalIdentity } from '../../../shared/agent-session/journal/agent-session-journal-identity' import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' import { sameProviderChild } from './structured-agent-session-provider-child' import { + STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS, STRUCTURED_AGENT_SESSION_STARTUP_LIMITS, type StructuredAgentSessionStartupAttempt, type StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { + measureStartup, + startupContractFailure, + startupDue, + trackedStartup, + type StructuredAgentSessionExpiredStartup, + type StructuredAgentSessionStartupProgress, + type StructuredAgentSessionStartupSettled, + type StructuredAgentSessionTrackedStartup +} from './structured-agent-session-startup-tracking' export { mintStructuredAgentSessionStartupAttempt } from './structured-agent-session-startup-attempt-mint' -export type StructuredAgentSessionExpiredStartup = { - sessionId: string - attemptId: string - child: StructuredAgentSessionProviderChildIdentity | null -} - -export type StructuredAgentSessionStartupSettled = { - agent: AgentSessionJournalIdentity['agent'] - outcome: 'ready' | 'ceiling' | 'ended' - durationMs: number -} +export type { + StructuredAgentSessionExpiredStartup, + StructuredAgentSessionStartupProgress, + StructuredAgentSessionStartupSettled +} from './structured-agent-session-startup-tracking' -export type StructuredAgentSessionStartupProgress = { - spawned: () => void -} - -type Tracked = { - attemptId: string - attempt: StructuredAgentSessionStartupAttempt | null - child: StructuredAgentSessionProviderChildIdentity | null - /** Absolute ceiling starts before any startup wait. */ - registeredAt: number - spawnedAt: number | null - timer: ReturnType | null - expired: boolean - /** Defer expiry reporting until aborted spawn settles; successful readiness overrides it. */ - expiry: StructuredAgentSessionStartupSettled | null - failure?: SubmissionRejectionFact - openError?: unknown - retry?: boolean - opening?: Promise - cancelOpen?: () => void - readiness: PromiseWithResolvers<'ready' | 'ended'> -} +type Tracked = StructuredAgentSessionTrackedStartup export class StructuredAgentSessionStartupAttempts { private readonly open = new Map() + /** The open settling each starting child's outcome; while it runs, the child's end waits for it. */ + private readonly owners = new Map() private disposed = false private readonly limits: StructuredAgentSessionStartupLimits private readonly now: () => number @@ -60,6 +46,8 @@ export class StructuredAgentSessionStartupAttempts { private readonly deps: { /** Expire outside serialization so blocked provider startup cannot delay cancellation. */ expire: (expired: StructuredAgentSessionExpiredStartup) => void + /** The adapter answered nothing within the grace after its exit; cancel like expiry. */ + breach?: (breached: StructuredAgentSessionExpiredStartup & { agent: string }) => void settled?: (settled: StructuredAgentSessionStartupSettled) => void limits?: Partial /** Use elapsed time; host clocks may be pinned. */ @@ -77,61 +65,12 @@ export class StructuredAgentSessionStartupAttempts { } pastCeiling(registeredAt: number): boolean { - return this.untilCeiling(registeredAt) <= 0 - } - - untilCeiling(registeredAt: number): number { - return registeredAt + this.limits.ceilingMs - this.now() - } - - childFailed( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity, - failure: SubmissionRejectionFact, - retry: boolean - ): void { - const tracked = this.open.get(sessionId) - if (tracked?.child && sameProviderChild(tracked.child, child)) { - Object.assign(tracked, { failure, retry }) - } - } - - /** The retained attempt survives child removal, but never records into its replacement. */ - observeOpen( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity, - cancel: () => void - ) { - const tracked = this.open.get(sessionId) - const completion = Promise.withResolvers() - const matches = tracked?.child && sameProviderChild(tracked.child, child) - if (matches) { - tracked.opening = completion.promise - tracked.cancelOpen = cancel - } - return { - failed: (failure: SubmissionRejectionFact, retry: boolean, openError: unknown): void => { - if (matches) { - Object.assign(tracked, { failure, retry, openError }) - } - }, - finish: (): void => completion.resolve() - } + return this.now() - registeredAt >= this.limits.ceilingMs } private register(sessionId: string, attemptId: string, registeredAt = this.now()): Tracked { this.end(sessionId) - const tracked: Tracked = { - attemptId, - attempt: null, - child: null, - registeredAt, - spawnedAt: null, - timer: null, - expired: false, - expiry: null, - readiness: Promise.withResolvers() - } + const tracked = trackedStartup(attemptId, registeredAt) this.open.set(sessionId, tracked) if (!this.disposed) { this.schedule(sessionId, tracked) @@ -149,6 +88,9 @@ export class StructuredAgentSessionStartupAttempts { ? begun : this.register(sessionId, attempt.attemptId) tracked.attempt = attempt + tracked.stopListening ??= attempt.events?.onRootExit?.(() => + this.rootExited(sessionId, tracked) + ) const current = (): boolean => this.open.get(sessionId) === tracked && !this.disposed return { spawned: () => { @@ -173,6 +115,39 @@ export class StructuredAgentSessionStartupAttempts { } } + /** The child's open claims its outcome; false if the child no longer starts on this clock. */ + own(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): boolean { + if (!this.tracking(sessionId, child)) { + return false + } + this.owners.set(sessionId, { generation: child.generation, fence: child.fence }) + return true + } + + disown(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): void { + if (this.ownsOpen(sessionId, child)) { + this.owners.delete(sessionId) + } + } + + ownsOpen(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): boolean { + const owner = this.owners.get(sessionId) + return !!owner && sameProviderChild(owner, child) + } + + /** The owner's verdict; readiness waiters and the start flight read it once readiness ends. */ + failed( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + failure: AgentSessionStartFailure + ): void { + const tracked = this.tracking(sessionId, child) + if (tracked) { + tracked.failure = failure + this.settle(sessionId, tracked, 'ended') + } + } + /** Published starts retain their clock; expire immediately if acquisition outlasted its limit. */ published( sessionId: string, @@ -197,13 +172,8 @@ export class StructuredAgentSessionStartupAttempts { /** Reject readiness that lost the race to expiry. */ onClock(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): boolean { - const tracked = this.open.get(sessionId) - return ( - !this.disposed && - !!tracked?.child && - !tracked.expired && - sameProviderChild(tracked.child, child) - ) + const tracked = this.tracking(sessionId, child) + return !this.disposed && !!tracked && !tracked.expired } ready(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): void { @@ -213,9 +183,34 @@ export class StructuredAgentSessionStartupAttempts { } } - childEnded(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): void { - const tracked = this.open.get(sessionId) - if (tracked?.child && sameProviderChild(tracked.child, child)) { + /** Settles once the adapter saw this starting child's own root exit; otherwise never. */ + rootExit(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): Promise { + return this.tracking(sessionId, child)?.rootExit.promise ?? new Promise(() => {}) + } + + /** Orca ending the child decides the start: a failure Orca recorded is final, any other close + * cancels it. An exit of its own leaves an open still settling the start, and its clock, until + * that open gives the outcome. */ + childEnded( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + ended: { byOrca: boolean; failure?: SubmissionRejectionFact; reason?: string | null } = { + byOrca: false + } + ): void { + const tracked = this.tracking(sessionId, child) + if (ended.failure) { + const cause = new Error(ended.reason ?? 'Orca stopped the agent while it started') + this.failed(sessionId, child, new AgentSessionStartFailure(cause, 'never', ended.failure)) + } else if (tracked && (ended.byOrca || !this.ownsOpen(sessionId, child))) { + this.settle(sessionId, tracked, 'ended') + } + } + + /** The owner ended without a verdict: replaced, cancelled or no longer starting. */ + released(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): void { + const tracked = this.tracking(sessionId, child) + if (tracked) { this.settle(sessionId, tracked, 'ended') } } @@ -236,11 +231,34 @@ export class StructuredAgentSessionStartupAttempts { this.disposed = true for (const tracked of this.open.values()) { tracked.readiness.resolve('ended') + tracked.stopListening?.() if (tracked.timer) { clearTimeout(tracked.timer) } } this.open.clear() + this.owners.clear() + } + + private tracking( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity + ): Tracked | undefined { + const tracked = this.open.get(sessionId) + return tracked?.child && sameProviderChild(tracked.child, child) ? tracked : undefined + } + + /** The deadline becomes the earlier of the original one and the grace after the exit. */ + private rootExited(sessionId: string, tracked: Tracked): void { + if (this.open.get(sessionId) !== tracked || this.disposed || tracked.rootExitAt !== null) { + return + } + tracked.rootExitAt = this.now() + tracked.rootExit.resolve() + if (tracked.timer) { + clearTimeout(tracked.timer) + } + this.schedule(sessionId, tracked) } private markSpawned(tracked: Tracked): void { @@ -248,7 +266,7 @@ export class StructuredAgentSessionStartupAttempts { } private schedule(sessionId: string, tracked: Tracked): void { - const due = tracked.registeredAt + this.limits.ceilingMs + const due = startupDue(tracked, this.limits.ceilingMs) tracked.timer = setTimeout(() => this.check(sessionId, tracked), Math.max(0, due - this.now())) tracked.timer.unref?.() } @@ -260,11 +278,25 @@ export class StructuredAgentSessionStartupAttempts { } if (this.now() - tracked.registeredAt >= this.limits.ceilingMs) { this.expire(sessionId, tracked, 'ceiling') + } else if ( + tracked.rootExitAt !== null && + this.now() - tracked.rootExitAt >= STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS + ) { + this.breach(sessionId, tracked) } else { this.schedule(sessionId, tracked) } } + /** Fires on a timer, so neither the adapter nor the session's serialized work can hold it. */ + private breach(sessionId: string, tracked: Tracked): void { + const agent = tracked.attempt?.identity.agent ?? 'agent' + tracked.failure = startupContractFailure(agent) + const breached = { sessionId, attemptId: tracked.attemptId, child: tracked.child, agent } + this.settle(sessionId, tracked, 'ended') + this.deps.breach?.(breached) + } + /** null means an already-expired spawn just published its child. */ private expire(sessionId: string, tracked: Tracked, reason: 'ceiling' | null): void { if (this.open.get(sessionId) !== tracked || this.disposed) { @@ -294,13 +326,7 @@ export class StructuredAgentSessionStartupAttempts { tracked: Tracked, outcome: StructuredAgentSessionStartupSettled['outcome'] ): StructuredAgentSessionStartupSettled | null { - return tracked.spawnedAt === null || !tracked.attempt - ? null - : { - agent: tracked.attempt.identity.agent, - outcome, - durationMs: Math.max(0, this.now() - tracked.spawnedAt) - } + return measureStartup(tracked, outcome, this.now()) } private report(settled: StructuredAgentSessionStartupSettled | null): void { @@ -317,6 +343,7 @@ export class StructuredAgentSessionStartupAttempts { private end(sessionId: string): void { const tracked = this.open.get(sessionId) tracked?.readiness.resolve('ended') + tracked?.stopListening?.() if (tracked?.timer) { clearTimeout(tracked.timer) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-tracking.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-tracking.ts new file mode 100644 index 000000000000..39ac73dc74cd --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-tracking.ts @@ -0,0 +1,98 @@ +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session/journal/agent-session-journal-identity' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' +import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' +import { + STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS, + StructuredAgentSessionStartupContractError, + type StructuredAgentSessionStartupAttempt +} from './structured-agent-session-startup-attempt-contract' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' + +export type StructuredAgentSessionExpiredStartup = { + sessionId: string + attemptId: string + child: StructuredAgentSessionProviderChildIdentity | null +} + +export type StructuredAgentSessionStartupSettled = { + agent: AgentSessionJournalIdentity['agent'] + outcome: 'ready' | 'ceiling' | 'ended' + durationMs: number +} + +export type StructuredAgentSessionStartupProgress = { + spawned: () => void +} + +export type StructuredAgentSessionTrackedStartup = { + attemptId: string + attempt: StructuredAgentSessionStartupAttempt | null + child: StructuredAgentSessionProviderChildIdentity | null + /** Absolute ceiling starts before any startup wait. */ + registeredAt: number + spawnedAt: number | null + timer: ReturnType | null + expired: boolean + /** Defer expiry reporting until aborted spawn settles; successful readiness overrides it. */ + expiry: StructuredAgentSessionStartupSettled | null + readiness: PromiseWithResolvers<'ready' | 'ended'> + /** When the adapter first saw this acquisition's own root exit; it shortens the deadline. */ + rootExitAt: number | null + rootExit: PromiseWithResolvers + stopListening?: () => void + /** Set before readiness settles when the start's owner or the contract guard failed it. */ + failure?: AgentSessionStartFailure +} + +export function trackedStartup( + attemptId: string, + registeredAt: number +): StructuredAgentSessionTrackedStartup { + return { + attemptId, + attempt: null, + child: null, + registeredAt, + spawnedAt: null, + timer: null, + expired: false, + expiry: null, + readiness: Promise.withResolvers(), + rootExitAt: null, + rootExit: Promise.withResolvers() + } +} + +/** The earlier of the original deadline and the grace after the acquisition's own root exit. */ +export function startupDue( + tracked: StructuredAgentSessionTrackedStartup, + ceilingMs: number +): number { + return Math.min( + tracked.registeredAt + ceilingMs, + (tracked.rootExitAt ?? Infinity) + STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS + ) +} + +/** A broken adapter contract fails its start for good, worded as the agent failing to start. */ +export function startupContractFailure(agent: string): AgentSessionStartFailure { + return new AgentSessionStartFailure( + new StructuredAgentSessionStartupContractError(agent), + 'never', + agentSessionFailureFact('providerStartFailed') + ) +} + +export function measureStartup( + tracked: StructuredAgentSessionTrackedStartup, + outcome: StructuredAgentSessionStartupSettled['outcome'], + now: number +): StructuredAgentSessionStartupSettled | null { + return tracked.spawnedAt === null || !tracked.attempt + ? null + : { + agent: tracked.attempt.identity.agent, + outcome, + durationMs: Math.max(0, now - tracked.spawnedAt) + } +} diff --git a/src/main/opencode/opencode-acp-connection.ts b/src/main/opencode/opencode-acp-connection.ts index de4e6a64b7b9..a38d6d942aff 100644 --- a/src/main/opencode/opencode-acp-connection.ts +++ b/src/main/opencode/opencode-acp-connection.ts @@ -109,7 +109,8 @@ class OpenCodeAcpConnection extends AcpAgentConnection { } catch { // Native history is best effort; never hide the ACP failure when it cannot be read. } - if (!failure || this.closed || failure.inputId === this.lastRecoveredInput) { + // An exit while reading ends the connection even while its output still drains. + if (!failure || this.closed || this.exited || failure.inputId === this.lastRecoveredInput) { throw error } this.lastRecoveredInput = failure.inputId diff --git a/src/main/pi/pi-scripted-child.test-fixture.ts b/src/main/pi/pi-scripted-child.test-fixture.ts index 05f16249195f..7eb8c23634a5 100644 --- a/src/main/pi/pi-scripted-child.test-fixture.ts +++ b/src/main/pi/pi-scripted-child.test-fixture.ts @@ -4,7 +4,10 @@ import { providerDiagnostic, withProviderDiagnostic } from '../../shared/agent-session/failures/agent-session-failure' -import type { JsonlRpcAgentConnectionOptions } from '../jsonl-rpc/agent-connection' +import { + JsonlRpcAgentExitError, + type JsonlRpcAgentConnectionOptions +} from '../jsonl-rpc/agent-connection' import { JsonlRpcResponseError, type JsonlRpcRecord } from '../jsonl-rpc/peer' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' import type { ScriptedAgentChild } from '../runtime/structured-agent-scripted-child.test-fixture' @@ -157,7 +160,11 @@ class ScriptedPiConnection implements PiRpcConnection { } exitHandshake(message: string): void { - const error = withProviderDiagnostic(new Error(message), providerDiagnostic(message, 'log')) + // As the real connection: an exit fails the pending calls with its own exit error. + const error = withProviderDiagnostic( + new JsonlRpcAgentExitError(message), + providerDiagnostic(message, 'log') + ) this.handshake.reject(error) this.exit(error) } diff --git a/src/main/pi/rpc-session-adapter.ts b/src/main/pi/rpc-session-adapter.ts index bffaf70d0a40..78030cab9981 100644 --- a/src/main/pi/rpc-session-adapter.ts +++ b/src/main/pi/rpc-session-adapter.ts @@ -18,6 +18,9 @@ import { buildPiRpcLaunch } from './rpc-launch' import type { PiRpcResolvedLaunch } from './rpc-launch-resolution' import { PiRpcSession, type PiRpcSessionDeps, type PiRpcConnection } from './rpc-session' import { dispatchPiRpcMessage } from './rpc-message-dispatch' +import { piAcquireFailure } from './rpc-startup-outcome' +import { openPiStart, watchPiStartExit } from './rpc-startup-open' +import { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' import { applyPiRpcSessionOption } from './rpc-options' import { supportsSupervisedProviderChildLocation } from '../provider-process/supervised-provider-child-location' @@ -33,6 +36,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { private readonly acquiring = new Set() private readonly starts = new ProviderAcquisitionStarts() private readonly retiring = new Set>() + private readonly endings = new AgentSessionStartEndings() constructor(private readonly deps: PiRpcSessionAdapterDeps) {} supportsLocation = supportsSupervisedProviderChildLocation @@ -81,6 +85,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { const fresh = !launch.sessionFile && !launch.forkFile session = new PiRpcSession(input, randomUUID(), spec, this.deps, launch, fresh) this.sessions.set(id, session) + watchPiStartExit(session, input, this.endings) this.starts.track(attempt, session.connection) const spawned = providerSpawnedProcessIdentity( input, @@ -108,7 +113,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { const result = await session.close(false).catch(() => null) const exited = piStartExitError(error, session.connection, result) if (exited) { - throw exited + throw piAcquireFailure(exited) } this.starts.retainFailed(id, session.connection) throw new AgentSessionAcquisitionExitUnprovenError(error) @@ -118,18 +123,8 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { } } - open: NonNullable = async (input) => { - const session = this.sessions.get(input.sessionId) - if ( - !session || - session.input.fence !== input.fence || - session.generation !== input.acquisitionGeneration || - session.connection.closed - ) { - throw new Error('Pi child changed before it opened') - } - return session.open(input.signal) - } + open: NonNullable = (input) => + openPiStart(this.sessions, this.endings, input) dispatch: StructuredAgentSessionAdapter['dispatch'] = async (input) => { const session = this.session(input.sessionId, input.fence) @@ -232,6 +227,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { } holdsDispatch = (id: string): boolean => this.sessions.get(id)?.turns.holdsDispatch ?? false heldSends = (id: string): ReadonlySet | undefined => this.sessions.get(id)?.heldSends + startupEnding = (id: string, generation: string) => this.endings.get(id, generation) holdsLiveProviderProcess(id: string, generation: string): boolean { const session = this.sessions.get(id) return session?.generation === generation && !session.connection.closed @@ -283,6 +279,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { void retirement.then(() => this.retiring.delete(retirement)) } async closeAll(): Promise { + this.endings.clear() await Promise.all( [...this.sessions.keys()].map((id) => stopAgentSessionProviderRoot(() => this.closeSession(id)) diff --git a/src/main/pi/rpc-session-startup.test.ts b/src/main/pi/rpc-session-startup.test.ts index 438671c19e6a..a68b81f7b70c 100644 --- a/src/main/pi/rpc-session-startup.test.ts +++ b/src/main/pi/rpc-session-startup.test.ts @@ -93,7 +93,8 @@ describe('Pi host awaited open', () => { command === 'get_state' ? Promise.resolve(unknown) : undefined }) await expect(openAt8(h, acquired.acquisitionGeneration)).rejects.toMatchObject({ - reason: 'notSignedIn' + retry: 'never', + cause: { reason: 'notSignedIn' } }) }) diff --git a/src/main/pi/rpc-session.ts b/src/main/pi/rpc-session.ts index 9aae0b4b97d5..0938316191d0 100644 --- a/src/main/pi/rpc-session.ts +++ b/src/main/pi/rpc-session.ts @@ -66,14 +66,15 @@ export class PiRpcSession { } private optionObservations = new PiRpcOptionObservations( this.optionProof, - () => !this.requestedClose && !this.failedCause && !this.connection.closed + () => !this.requestedClose && !this.failure && !this.connection.closed ) readonly startup: PiRpcSessionStartup readonly history = PiHistoryCapture.forSession(this, () => this.deps) openedLink: AgentSessionProviderHandleLink | null = null requestedClose = false private publishedExit = false - private failedCause?: Error + /** What ended this child, once something did. */ + failure?: Error private exitDelivery: Promise = Promise.resolve() private releaseAfterExit?: () => void private readonly reading = new ReaderPauseHolds<'lane' | 'journal'>(() => this.connection) @@ -163,7 +164,7 @@ export class PiRpcSession { this.lane.apply(this.turns.context.setModel(state.model ?? undefined, Date.now())) this.optionObservations.seed(state) }, - () => this.failedCause, + () => this.failure, () => this.optionObservations.prepare() ) } @@ -211,7 +212,7 @@ export class PiRpcSession { } fail(error: Error): void { - this.failedCause ??= error + this.failure ??= error this.deps.logger.error('Pi RPC session failed', { scope: 'pi-rpc', sessionId: this.input.identity.sessionId, @@ -264,7 +265,7 @@ export class PiRpcSession { this.publishedExit = true const observedAt = Date.now() this.input.events?.observeExit?.(this.heldSends, observedAt) - const cause = (this.failedCause ??= error) + const cause = (this.failure ??= error) this.dialogs.cancelAll() this.turns.end() // The lane is sealed once finalized, so the end event travels with the final transfer. diff --git a/src/main/pi/rpc-startup-open.ts b/src/main/pi/rpc-startup-open.ts new file mode 100644 index 000000000000..99dde72ef392 --- /dev/null +++ b/src/main/pi/rpc-startup-open.ts @@ -0,0 +1,58 @@ +import type { + StructuredAgentSessionAcquireInput, + StructuredAgentSessionAdapter +} from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import type { StructuredAgentSessionProviderOpenInput } from '../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' +import { JsonlRpcAgentExitError } from '../jsonl-rpc/agent-connection' +import type { PiRpcSession } from './rpc-session' +import { piStartFailure } from './rpc-startup-outcome' + +/** Takes this child's own exit when first seen, so a start settling meanwhile cannot miss it; a + * protocol failure before it stays the cause. */ +export function watchPiStartExit( + session: PiRpcSession, + input: Pick, + endings: AgentSessionStartEndings +): void { + const sessionId = session.input.identity.sessionId + endings.forget(sessionId) + session.connection.onExit((requested) => { + if (!requested) { + input.events?.observeRootExit?.() + const exited = new JsonlRpcAgentExitError('Pi exited while starting') + endings.record(sessionId, session.generation, piStartFailure(session.failure ?? exited)) + } + }) +} + +/** A start's open answers with Pi's verdict; a child that already ended answers with how it + * ended, and an answer its own exit followed is still worded as that exit. */ +export async function openPiStart( + sessions: Map, + endings: AgentSessionStartEndings, + input: StructuredAgentSessionProviderOpenInput +): ReturnType> { + const session = sessions.get(input.sessionId) + try { + if ( + !session || + session.input.fence !== input.fence || + session.generation !== input.acquisitionGeneration + ) { + throw ( + endings.get(input.sessionId, input.acquisitionGeneration) ?? + new Error('Pi child changed before it opened') + ) + } + if (session.connection.closed) { + throw session.failure ?? new Error('Pi child ended before it opened') + } + return await session.open(input.signal) + } catch (error) { + throw piStartFailure( + error, + session?.connection.rootVerdict === 'exited' && !session.requestedClose + ) + } +} diff --git a/src/main/pi/rpc-startup-outcome.ts b/src/main/pi/rpc-startup-outcome.ts new file mode 100644 index 000000000000..47313f701f0f --- /dev/null +++ b/src/main/pi/rpc-startup-outcome.ts @@ -0,0 +1,25 @@ +import { + agentSessionAcquireFailure, + agentSessionStartFailure, + type AgentSessionStartAnswer, + type AgentSessionStartFailure +} from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { JsonlRpcAgentExitError } from '../jsonl-rpc/agent-connection' +import { JsonlRpcResponseError } from '../jsonl-rpc/peer' + +/** Pi declares no temporary startup answers. Only its process's own exit, once its output + * drained, leaves a start unanswered; a closed stream alone proves no exit. */ +export const piStartupAnswer: AgentSessionStartAnswer = (cause) => { + if (cause instanceof JsonlRpcResponseError) { + return 'final' + } + return cause instanceof JsonlRpcAgentExitError ? 'exited' : undefined +} + +export function piStartFailure(error: unknown, exited = false): AgentSessionStartFailure { + return agentSessionStartFailure(error, piStartupAnswer, exited) +} + +export function piAcquireFailure(error: unknown): unknown { + return agentSessionAcquireFailure(error, piStartupAnswer) +} diff --git a/src/main/provider-process/provider-exit-output-drain.ts b/src/main/provider-process/provider-exit-output-drain.ts new file mode 100644 index 000000000000..995bb0bf61e5 --- /dev/null +++ b/src/main/provider-process/provider-exit-output-drain.ts @@ -0,0 +1,39 @@ +import type { EventEmitter } from 'node:events' +import type { Readable } from 'node:stream' + +/** How long an exited child's output may still deliver answers before its pending calls fail. */ +export const PROVIDER_EXIT_OUTPUT_DRAIN_MS = 1_000 + +/** Process exit can be observed before stdout is read to its end, so an answer written just + * before exit would otherwise lose to the exit. Runs `finish` once stdout ends or closes, the + * child reports all its stdio closed, or the bound passes while a descendant still holds the + * pipe. Returns a way to finish now. */ +export function afterProviderOutputDrains( + stdout: Readable, + finish: () => void, + child?: EventEmitter +): () => void { + let finished = false + let timer: ReturnType | undefined + const done = (): void => { + if (finished) { + return + } + finished = true + clearTimeout(timer) + stdout.removeListener('end', done) + stdout.removeListener('close', done) + child?.removeListener('close', done) + finish() + } + if (stdout.destroyed || stdout.readableEnded) { + done() + return done + } + stdout.once('end', done) + stdout.once('close', done) + child?.once('close', done) + timer = setTimeout(done, PROVIDER_EXIT_OUTPUT_DRAIN_MS) + timer.unref?.() + return done +} diff --git a/src/main/provider-process/provider-output-end.ts b/src/main/provider-process/provider-output-end.ts new file mode 100644 index 000000000000..2edd23b40ad7 --- /dev/null +++ b/src/main/provider-process/provider-output-end.ts @@ -0,0 +1,59 @@ +import type { Readable, Writable } from 'node:stream' +import { afterProviderOutputDrains } from './provider-exit-output-drain' +import { detachProviderStreamErrorHandler } from './provider-stdio-error-boundary' + +/** The agent stopped reading what Orca writes. Nothing more is written, but calls it may already + * have answered settle only once its output ends or the drain bound passes. */ +export class ProviderOutputEnd { + error?: Error + private detached = false + + constructor( + private readonly output: Writable, + private readonly ended: (error: Error) => void, + private readonly closedError: () => Error + ) { + output.on('close', this.onClosed) + output.on('finish', this.onClosed) + output.on('error', this.onError) + } + + /** False when the output already ended; the first end is the one its calls settle with. */ + begin(error: Error): boolean { + if (this.error) { + return false + } + this.error = error + this.detach() + return true + } + + /** `close` runs now when no call waits, else once `wait` (by default the output drain) ends. */ + settle( + waiting: boolean, + input: Readable, + close: () => void, + wait?: (finish: () => void) => void + ): void { + if (!waiting) { + close() + } else if (wait) { + wait(close) + } else { + afterProviderOutputDrains(input, close) + } + } + + detach(): void { + if (this.detached) { + return + } + this.detached = true + this.output.removeListener('close', this.onClosed) + this.output.removeListener('finish', this.onClosed) + detachProviderStreamErrorHandler(this.output, this.onError) + } + + private readonly onClosed = (): void => this.ended(this.closedError()) + private readonly onError = (error: Error): void => this.ended(error) +} diff --git a/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts b/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts index 5c3a4360417b..150f60ee1b8f 100644 --- a/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts +++ b/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts @@ -11,6 +11,8 @@ import { setStructuredAgentSessionHost } from '../../../native-chat/agent-sessio import type { RpcContext } from '../core' import { AgentSessionPreSpawnError } from '../../../native-chat/agent-session-wire/structured-agent-session-adapter' import { CodexAppServerRequestError } from '../../../codex/app-server/codex-app-server-request-error' +import { CodexAppServerExitError } from '../../../codex/app-server/codex-app-server-exit-error' +import { codexStartFailure } from '../../../codex/app-server/codex-startup-outcome' import { executeAgentLaunch } from '../../../agent-launch/agent-launch-executor' import { agentLaunchSurfaceFactory } from './agent-launch-surfaces' import * as structuredCreate from './structured-agent-session-create' @@ -144,8 +146,10 @@ async function endChild(sessionId = SESSION) { reason: 'exited during initialize', cause: 'unexpected-exit' }) - // A real connection rejects its outstanding open once the child exits. - openEnded.get(current.acquisitionGeneration)?.(new Error('codex app-server connection ended')) + // The real adapter answers its outstanding open with the drained exit once the child exits. + openEnded.get(current.acquisitionGeneration)?.( + codexStartFailure(new CodexAppServerExitError('codex app-server connection ended')) + ) openEnded.delete(current.acquisitionGeneration) await vi.waitFor(() => expect(rig.host.collaboratorsForTests().sessions.get(sessionId)?.child).toBeNull() @@ -249,11 +253,13 @@ it.each(['spawn', 'open'] as const)( ) } else { vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValueOnce( - new CodexAppServerRequestError( - 'initialize', - -32001, - 'Server overloaded; retry later.', - 'Server overloaded; retry later.' + codexStartFailure( + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) ) ) } From 185ee4802e6c2742b28d26d024e91e6fd86ba96d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:00:05 -0800 Subject: [PATCH 15/21] fix(agent-session): bound a failed start's lifetime and keep Orca's own refusals final - A start's ending is kept only while its owner can read it: admitted when the acquisition starts, retired when the owner finishes, the acquisition fails, or the session is released or closed. An exit after ready records nothing. - Codex takes its own exit as the start's wake from the moment the process is seen, before spawn bookkeeping, and a failed acquire keeps that exit as cause. - Once the handle or rewind write began, an unanswered death no longer decides the start: the write's refusal stays final, and the exit grace bounds the wait. - Stop, the deadline and the exit grace release the start's callers at once; the failure row is queued first but never awaited by them. - An ACP connection Orca closed before the exit keeps that cause, so cleanup's exit is not read as the agent ending unanswered. - A request deadline keeps its words when an exit is seen around it. - A direct attach whose exit settled first keeps the exit's words. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/main/acp/acp-agent-connection.test.ts | 3 +- src/main/acp/acp-agent-connection.ts | 17 +- .../acp/acp-session-reopen-failure.test.ts | 42 +++- .../acp/acp-structured-child-lifecycle.ts | 12 +- .../acp/acp-structured-session-adapter.ts | 26 +- .../claude-structured-session-adapter.ts | 17 +- .../codex/codex-structured-session-acquire.ts | 15 +- .../codex/codex-structured-session-adapter.ts | 2 + .../codex/codex-structured-session-state.ts | 1 - .../codex-structured-session-teardown.ts | 1 + .../codex-structured-start-endings.test.ts | 45 ++++ ...structured-agent-session-adapter-router.ts | 5 + .../structured-agent-session-adapter.ts | 2 + ...red-agent-session-conversation-lifetime.ts | 5 +- ...uctured-agent-session-failure-text.test.ts | 7 +- .../structured-agent-session-failure-text.ts | 6 +- ...ctured-agent-session-provider-open.test.ts | 39 +++ .../structured-agent-session-provider-open.ts | 8 + ...ructured-agent-session-provider-started.ts | 33 ++- ...structured-agent-session-start-attempts.ts | 223 ++++++++++-------- .../structured-agent-session-start-failure.ts | 35 ++- ...agent-session-startup-attempt-host.test.ts | 7 +- ...red-agent-session-startup-liveness.test.ts | 182 ++++++++++++++ src/main/pi/rpc-session-adapter.ts | 4 + src/main/pi/rpc-startup-open.ts | 2 +- 25 files changed, 593 insertions(+), 146 deletions(-) create mode 100644 src/main/codex/codex-structured-start-endings.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts diff --git a/src/main/acp/acp-agent-connection.test.ts b/src/main/acp/acp-agent-connection.test.ts index 2659b0080818..136770f35f85 100644 --- a/src/main/acp/acp-agent-connection.test.ts +++ b/src/main/acp/acp-agent-connection.test.ts @@ -352,7 +352,8 @@ describe('ACP process-owning connection', () => { expect(() => child.emit('exit', 0, null)).not.toThrow() expect(early).toHaveBeenCalledExactlyOnceWith( { code: 0, signal: null, processless: false }, - false + false, + undefined ) expect(() => connection.onExit(failing)).not.toThrow() const late = vi.fn() diff --git a/src/main/acp/acp-agent-connection.ts b/src/main/acp/acp-agent-connection.ts index fb51e37d37ee..0e1ff1561901 100644 --- a/src/main/acp/acp-agent-connection.ts +++ b/src/main/acp/acp-agent-connection.ts @@ -49,6 +49,10 @@ export function createAcpAgentConnection( type AcpConnectionLifecycle = { closing: boolean error?: Error + /** The process's own failure, such as its spawn; it is no refusal by Orca. */ + processError?: Error + /** Why the connection closed before the process exited, unless the process itself failed. */ + lost?: Error finishExit?: () => void outputEndTimer?: ReturnType } @@ -149,7 +153,10 @@ export class AcpAgentConnection extends AcpSessionRuntime { child.once('spawn', done) child.once('error', done) }) - child.on('error', (error) => super.close(error)) + child.on('error', (error) => { + lifecycle.processError ??= error + super.close(error) + }) signedOut = () => { signIn?.answered?.(false) super.close(new AcpSignInPromptError('The agent asked to sign in in a browser')) @@ -159,6 +166,7 @@ export class AcpAgentConnection extends AcpSessionRuntime { super.close(new AcpConnectionClosedError('The agent printed a sign-in link Orca cannot read')) child.stderr.on('error', (error) => super.close(error)) managed.onExit((exit) => { + lifecycle.lost = lifecycle.error === lifecycle.processError ? undefined : lifecycle.error const expected = lifecycle.closing const finish = (): void => { clearTimeout(lifecycle.outputEndTimer) @@ -225,11 +233,12 @@ export class AcpAgentConnection extends AcpSessionRuntime { return this.managed.exitStatus } - /** `requested`: Orca was closing the agent when it exited. */ - onExit(listener: (exit: ProviderProcessExit, requested?: boolean) => void): void { + /** `requested`: Orca was closing the agent when it exited. `lost`: why the connection had + * already closed before the exit, such as Orca refusing what the agent wrote. */ + onExit(listener: (exit: ProviderProcessExit, requested?: boolean, lost?: Error) => void): void { this.managed.onExit((exit) => { try { - listener(exit, this.lifecycle.closing) + listener(exit, this.lifecycle.closing, this.lifecycle.lost) } catch (failure) { this.diagnoseLifecycle(`ACP exit listener failed: ${String(failure)}`) } diff --git a/src/main/acp/acp-session-reopen-failure.test.ts b/src/main/acp/acp-session-reopen-failure.test.ts index 16a19955047d..21da77a1466a 100644 --- a/src/main/acp/acp-session-reopen-failure.test.ts +++ b/src/main/acp/acp-session-reopen-failure.test.ts @@ -1,7 +1,11 @@ import { openAcpAdapterRig } from './acp-structured-adapter.test-support' import { closeProviderTimelineRigs } from '../native-chat/agent-session-timeline/provider-timeline-assembler-test-support' import { afterEach, expect, it, vi } from 'vitest' -import { AcpAgentError } from './acp-errors' +import { AcpAgentError, AcpRequestTimeoutError } from './acp-errors' +import { openAcpStart } from './acp-structured-start-open' +import { AgentSessionStartFailure } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { agentSessionFailureFact } from '../../shared/agent-session/failures/agent-session-failure' +import { agentSessionFailureWords } from '../../shared/agent-session/failures/agent-session-failure-words' import { acpReopenTakeover } from './acp-session-reopen-failure' it.each(['ServerUnavailable', 'CatalogNotReady'])( @@ -51,3 +55,39 @@ it('propagates a temporary load envelope before takeover or a dead-child asserti expect(rig.sent('session/load')[0].params).toMatchObject({ sessionId: 'saved-id' }) await rig.adapter.closeAll() }) + +it("keeps a load deadline's words when the agent exited while its output drained", async () => { + const timeout = new AcpRequestTimeoutError('session/load', 300_000) + // The agent exited unanswered at 299.5 s; the request's own deadline fired while its output drained. + const child = { + fence: 3, + acquisitionGeneration: 'g1', + ended: false, + closeRequested: false, + connection: { exited: true }, + spec: { agent: 'opencode', dialect: 'opencode' }, + open: async () => { + throw timeout + } + } + const failure = await openAcpStart( + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: openAcpStart reads only the starting child's fields set above. + { sessions: new Map([['s1', child]]) } as unknown as Parameters[0], + { sessionId: 's1', fence: 3, acquisitionGeneration: 'g1', signal: new AbortController().signal } + ).catch((error: unknown) => error) + expect(failure).toMatchObject({ + retry: 'never', + fact: { kind: 'startFailed', requestTimeout: { method: 'session/load', timeoutMs: 300_000 } } + }) + if (!(failure instanceof AgentSessionStartFailure)) { + throw new Error('the open did not fail with its verdict') + } + expect(agentSessionFailureWords(failure.fact, { surface: 'rejection' })).toEqual( + agentSessionFailureWords( + agentSessionFailureFact('startFailed', { + requestTimeout: { method: 'session/load', timeoutMs: 300_000 } + }), + { surface: 'rejection' } + ) + ) +}) diff --git a/src/main/acp/acp-structured-child-lifecycle.ts b/src/main/acp/acp-structured-child-lifecycle.ts index 4459c05b794e..9c705930bd5a 100644 --- a/src/main/acp/acp-structured-child-lifecycle.ts +++ b/src/main/acp/acp-structured-child-lifecycle.ts @@ -189,12 +189,18 @@ export class AcpStructuredChildren { } } - /** Taken when the process exit is first seen, so a start settling meanwhile cannot miss it. */ - recordExit(sessionId: string, generation: string, connection: AcpStructuredConnection): void { + /** Taken when the process exit is first seen, so a start settling meanwhile cannot miss it. A + * connection Orca closed first keeps that cause; only a bare exit is the agent's own. */ + recordExit( + sessionId: string, + generation: string, + connection: AcpStructuredConnection, + lost?: Error + ): void { const detail = connection.stderrTail() || `${this.deps.spec.agent} ACP agent ended while starting` const exited = withProviderSpawnAbsence( - new AcpAgentExitedError(detail, connection.stderrTail()), + lost ?? new AcpAgentExitedError(detail, connection.stderrTail()), connection.spawnAbsence ) this.endings.record(sessionId, generation, acpStartFailure(exited)) diff --git a/src/main/acp/acp-structured-session-adapter.ts b/src/main/acp/acp-structured-session-adapter.ts index 3cad7b0f5a75..9ac78a39244f 100644 --- a/src/main/acp/acp-structured-session-adapter.ts +++ b/src/main/acp/acp-structured-session-adapter.ts @@ -65,7 +65,6 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { async acquire(input: StructuredAgentSessionAcquireInput): Promise { const sessionId = input.identity.sessionId - this.children.endings.forget(sessionId) const gate = this.deps.spec.launchGate?.enter(input.signal) const attempt = this.children.starts.begin(gate?.signal ?? input.signal) try { @@ -91,6 +90,10 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { ): Promise { const sessionId = input.identity.sessionId const generation = this.deps.mintGeneration?.() ?? randomUUID() + this.children.endings.admit(sessionId, generation) + // Orca closing the connection first, such as on a sign-in line it cannot read, is the start's + // cause; the exit its cleanup then causes is not the agent ending unanswered. + let lostFirst: Error | undefined try { const { acquisition, session } = await acquireAcpStructuredSession({ acquire: input, @@ -98,10 +101,11 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { generation, abandoned: () => attempt.signal.aborted, track: (connection) => { - connection.onExit((_exit, requested) => { + connection.onExit((_exit, requested, lost) => { + lostFirst ??= lost if (!requested) { input.events?.observeRootExit?.() - this.children.recordExit(sessionId, generation, connection) + this.children.recordExit(sessionId, generation, connection, lost) } }) this.children.starts.track(attempt, connection) @@ -123,13 +127,14 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { this.children.sessions.set(sessionId, session) return acquisition } catch (error) { + this.children.endings.retire(sessionId, generation) const { connection } = attempt if (connection && error instanceof AcpConnectionClosedError) { // Wait briefly for process exit so startup failures include final diagnostics. await waitForAcpExit(connection, this.stopGraceMs(), attempt.signal) } // Capture spontaneous exit before close can classify it as requested. - const exitedOnItsOwn = connection?.exited === true && !attempt.signal.aborted + const exitedOnItsOwn = connection?.exited === true && !attempt.signal.aborted && !lostFirst if (connection && !(await connection.close().catch(() => false))) { // Retain failed cleanup for retry before another child can spawn. this.children.starts.retainFailed(sessionId, connection) @@ -165,6 +170,9 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { startupEnding: NonNullable = (sessionId, generation) => this.children.endings.get(sessionId, generation) + retireStartup: NonNullable = (sessionId, generation) => + this.children.endings.retire(sessionId, generation) + async dispatch(input: Parameters[0]): Promise { const lost = this.children.sessions.get(input.sessionId) if (lost && lost.journalClosed !== null) { @@ -298,9 +306,13 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { backgroundTaskStops = (sessionId: string) => acpChildStopCapabilities(this.children.sessions.get(sessionId)) - closeSession = (sessionId: string): Promise => this.children.close(sessionId) - disposeSession = (sessionId: string): Promise => this.children.close(sessionId) - releaseAcquisition = (input: { sessionId: string }) => this.children.close(input.sessionId) + // A released or closed session's start has no reader left for its ending. + closeSession = (sessionId: string): Promise => { + this.children.endings.retire(sessionId) + return this.children.close(sessionId) + } + disposeSession = this.closeSession + releaseAcquisition = (input: { sessionId: string }) => this.closeSession(input.sessionId) hasOwnedAcquisition = (sessionId: string): boolean => this.children.sessions.has(sessionId) || this.children.starts.failedSessionIds().includes(sessionId) diff --git a/src/main/claude/structured/claude-structured-session-adapter.ts b/src/main/claude/structured/claude-structured-session-adapter.ts index 4d21b87a2989..26b7bf1fd543 100644 --- a/src/main/claude/structured/claude-structured-session-adapter.ts +++ b/src/main/claude/structured/claude-structured-session-adapter.ts @@ -74,7 +74,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda acquire = (input: StructuredAgentSessionAcquireInput) => { this.settledExitErrors.delete(input.identity.sessionId) - this.endings.forget(input.identity.sessionId) + this.endings.retire(input.identity.sessionId) const record = this.deps.namingRecord?.(input.identity.sessionId) const naming = this.naming?.acquire(input, record) return acquireClaudeSession({ @@ -100,10 +100,12 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda settleExit: (sessionId, exit) => settleClaudeUnexpectedExit(this.exitLifecycle, sessionId, exit) } - }).catch((error: unknown) => { - naming?.failed() - throw claudeAcquireFailure(error) }) + .then(this.endings.admitting(input.identity.sessionId)) + .catch((error: unknown) => { + naming?.failed() + throw claudeAcquireFailure(error) + }) } private deliver(attempt: ClaudeAcquisitionAttempt, sessionId: string, event: () => void): void { @@ -234,12 +236,15 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda }) readOptions = (input: { sessionId: string; fence: number }) => readClaudeStructuredSessionOptions(this.session(input.sessionId)) - releaseAcquisition = (input: { sessionId: string }): Promise => - this.afterClose(input.sessionId, () => this.releaseProviderSession(input.sessionId)) + releaseAcquisition = (input: { sessionId: string }): Promise => { + this.endings.retire(input.sessionId) + return this.afterClose(input.sessionId, () => this.releaseProviderSession(input.sessionId)) + } acknowledgeSessionRelease = (sessionId: string): void => { this.settledExitErrors.delete(sessionId) } startupEnding = (sessionId: string, generation: string) => this.endings.get(sessionId, generation) + retireStartup = (id: string, generation: string) => this.endings.retire(id, generation) hasOwnedAcquisition = (sessionId: string): boolean => this.acquisitions.get(sessionId) !== undefined || this.sessions.has(sessionId) || diff --git a/src/main/codex/codex-structured-session-acquire.ts b/src/main/codex/codex-structured-session-acquire.ts index f821002be50d..82d1d687af3d 100644 --- a/src/main/codex/codex-structured-session-acquire.ts +++ b/src/main/codex/codex-structured-session-acquire.ts @@ -77,6 +77,9 @@ export async function acquireCodexStructuredSession(input: { let account: AgentSessionAccountKind | undefined // Frame row identities include this generation. const acquisitionGeneration = mintCodexAcquisitionGeneration(deps) + acquisitions.endings.admit(sessionId, acquisitionGeneration) + // Codex ending on its own before acquire returns; acquire fails with it, not a generic error. + let exitedOnItsOwn: Error | undefined const translator = createCodexSessionJournalTranslator({ sink: acquireInput.events, account: () => account, @@ -164,12 +167,14 @@ export async function acquireCodexStructuredSession(input: { onExit: (error, exit) => { acquisition.appliedSettings.end() try { - // Orca's own close or cancel ends no start; Codex ending on its own does. - if (attempt.acquisitionGeneration && !exit?.expected && !attempt.cancelled) { + // Orca's own close or cancel ends no start; Codex ending on its own does, from the + // moment its process is seen, before spawn bookkeeping finishes. + if (!exit?.expected && !attempt.cancelled) { + exitedOnItsOwn ??= error acquireInput.events?.observeRootExit?.() acquisitions.endings.record( sessionId, - attempt.acquisitionGeneration, + acquisitionGeneration, codexStartFailure(error) ) } @@ -232,6 +237,9 @@ export async function acquireCodexStructuredSession(input: { acquisitions.assertCurrent(sessionId, attempt) const process = await spawnIdentity.read(connection.pid) acquisitions.assertCurrent(sessionId, attempt) + if (connection.closed && exitedOnItsOwn) { + throw exitedOnItsOwn + } assertCodexConnectionOpen(connection, sessionId) attempt.acquisitionGeneration = acquisitionGeneration let disposed = false @@ -269,6 +277,7 @@ export async function acquireCodexStructuredSession(input: { ) return { process, acquisitionGeneration } } catch (error) { + acquisitions.endings.retire(sessionId, acquisitionGeneration) if (sessions.get(sessionId)?.connection !== acquisition.connection) { return closeFailedCodexAcquisition({ sessionId, diff --git a/src/main/codex/codex-structured-session-adapter.ts b/src/main/codex/codex-structured-session-adapter.ts index 7d02fe579fef..c1307600544f 100644 --- a/src/main/codex/codex-structured-session-adapter.ts +++ b/src/main/codex/codex-structured-session-adapter.ts @@ -287,6 +287,8 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap startupEnding = (sessionId: string, generation: string) => this.acquisitions.endings.get(sessionId, generation) + retireStartup = (sessionId: string, generation: string) => + this.acquisitions.endings.retire(sessionId, generation) compact: NonNullable = (input) => compactCodexStructuredSession(this.session(input.sessionId), input, this.deps.requestTimeoutMs) diff --git a/src/main/codex/codex-structured-session-state.ts b/src/main/codex/codex-structured-session-state.ts index 79da3c1b73cd..b23244773a0a 100644 --- a/src/main/codex/codex-structured-session-state.ts +++ b/src/main/codex/codex-structured-session-state.ts @@ -272,7 +272,6 @@ export class CodexAcquisitionRegistry { throw new Error('codex structured session adapter is closing') } const previousAttempt = this.attempts.get(sessionId) - this.endings.forget(sessionId) const attempt = createCodexAcquisitionAttempt() this.attempts.set(sessionId, attempt) return { previousAttempt, attempt } diff --git a/src/main/codex/codex-structured-session-teardown.ts b/src/main/codex/codex-structured-session-teardown.ts index 102c56d4fced..caad90e438e8 100644 --- a/src/main/codex/codex-structured-session-teardown.ts +++ b/src/main/codex/codex-structured-session-teardown.ts @@ -24,6 +24,7 @@ export class CodexStructuredSessionTeardown { constructor(private readonly deps: CodexStructuredSessionTeardownDeps) {} close = async (sessionId: string, observeClose?: AgentSessionCloseObserver): Promise => { + this.deps.acquisitions.endings.retire(sessionId) const connection = this.deps.sessions.get(sessionId)?.connection ?? this.deps.acquisitions.get(sessionId)?.window.connection diff --git a/src/main/codex/codex-structured-start-endings.test.ts b/src/main/codex/codex-structured-start-endings.test.ts new file mode 100644 index 000000000000..5a55b59fb687 --- /dev/null +++ b/src/main/codex/codex-structured-start-endings.test.ts @@ -0,0 +1,45 @@ +import { expect, it } from 'vitest' +import { CodexAppServerExitError } from './app-server/codex-app-server-exit-error' +import { adapterFor, fakeCodex, identityFor } from './codex-structured-session-adapter-fixture' + +it("keeps a start's ending only while that start can still read it", async () => { + const codex = fakeCodex() + const adapter = adapterFor(codex) + const exit = () => + codex.connections + .at(-1)! + .handlers.onExit?.(new CodexAppServerExitError('codex app-server connection ended'), { + expected: false + }) + const acquire = async (id: string) => { + const acquired = await adapter.acquire({ + identity: identityFor(id), + fence: 1, + spawnToken: `spawn-${id}` + }) + return { id, generation: acquired.acquisitionGeneration } + } + const retained = (starts: { id: string; generation: string }[]) => + starts.filter(({ id, generation }) => adapter.startupEnding(id, generation)) + + // Each start's own exit is readable until its owner is done, it is released, or it is closed. + const ended: { id: string; generation: string }[] = [] + for (const id of ['owner-done', 'released', 'closed']) { + ended.push(await acquire(id)) + codex.connections.at(-1)!.closed = true + exit() + } + expect(retained(ended)).toHaveLength(3) + adapter.retireStartup(ended[0].id, ended[0].generation) + await adapter.releaseAcquisition({ sessionId: ended[1].id }) + await adapter.disposeSession(ended[2].id) + expect(retained(ended)).toEqual([]) + + // A start its owner made ready leaves nothing behind when it later exits. + const ready = await acquire('ready') + adapter.retireStartup(ready.id, ready.generation) + codex.connections.at(-1)!.closed = true + exit() + expect(retained([ready])).toEqual([]) + await adapter.closeAll() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts index 1e413c478fc9..b4398d46ff65 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts @@ -162,6 +162,11 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi .get(sessionId) ?.registration.adapter.startupEnding?.(sessionId, acquisitionGeneration) + retireStartup = (sessionId: string, acquisitionGeneration: string) => + this.routes + .get(sessionId) + ?.registration.adapter.retireStartup?.(sessionId, acquisitionGeneration) + awaitStoppedRequestEnd = async (sessionId: string, stoppedAt: number) => this.liveOwnerOrNull(sessionId)?.awaitStoppedRequestEnd?.(sessionId, stoppedAt) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index 447262a2aec9..29637024d53f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -274,6 +274,8 @@ export type StructuredAgentSessionAdapter = StructuredAgentSessionAdapterStop & sessionId: string, acquisitionGeneration: string ): AgentSessionStartFailure | undefined + /** The start's owner finished: nothing reads this acquisition's ending any more. */ + retireStartup?(sessionId: string, acquisitionGeneration: string): void /** Undefined preserves the client catalog when the provider reports none. */ readCommands?(sessionId: string): AgentSessionSlashCommand[] | undefined atRestCommands?: StructuredAgentSessionAtRestCommands diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 22d259321736..3b4e02cfdcc6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -241,11 +241,14 @@ export function createStructuredAgentSessionConversationLifetime( if (!host.context().runtimeState.startFlights.ownsChild(sessionId, failed)) { const record = deps().store.getRecord(sessionId) const wording = structuredAgentSessionFailureWordsContext(record) + // A start whose own exit settled first keeps that exit's wording; only a request + // deadline, the more specific explanation, outranks it. The verdict alone decides retry. + const cause = exited && !failure.requestTimeout ? { exit: ended?.failure } : { failure } await recordStructuredAgentSessionStartFailure( { journal: session.journal, fence: failed.fence }, { startKey: failed.generation, - ...structuredAgentSessionStartFailure({ failure }, wording) + ...structuredAgentSessionStartFailure(cause, wording) }, oldestQueuedSubmission(session)?.clientMessageId ?? null ).catch((writeError: unknown) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.test.ts index 3fe9e3775e62..23f81d9571c2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.test.ts @@ -32,7 +32,7 @@ const ORCA_INTERNAL = /agent_session_|execution_owner|provider_[a-z_]+|[0-9a-f]{8}-[0-9a-f]{4}-|[/\\][\w.-]+[/\\]|Error:|ENOENT/ describe('structuredAgentSessionStartFailure', () => { - it('names a provider request that ran out of time, through wrappers, unless the provider exited', () => { + it('names a provider request that ran out of time, through wrappers and an exit seen around it', () => { const timedOut = withProviderRequestTimeout(new Error('thread/start exceeded 30000ms'), { method: 'thread/start', timeoutMs: 30_000 @@ -56,7 +56,10 @@ describe('structuredAgentSessionStartFailure', () => { ) expect( structuredAgentSessionStartFailure({ error: withObservedProviderExit(wrapped) }).rejection - ).toEqual({ kind: 'providerStartFailed' }) + ).toEqual({ + kind: 'startFailed', + requestTimeout: { method: 'thread/start', timeoutMs: 30_000 } + }) }) it('names a saved Arguments option in start and restart failures without exposing its value', () => { const error = new StructuredAgentArgumentsError( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts index 9ca67d792cc6..027b51447cf1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts @@ -149,7 +149,9 @@ export function providerStartupFailureFact(cause?: unknown): SubmissionRejection if (preparation) { return agentSessionFailureFact('launchPreparationFailed', { detail: preparation }) } - const exited = providerExitObserved(cause) + // A request deadline is the most specific explanation: an exit seen around it never replaces it. + const requestTimeout = providerRequestTimeoutOf(cause) + const exited = !requestTimeout && providerExitObserved(cause) return agentSessionFailureFact(exited ? 'providerStartFailed' : 'startFailed', { // A typed refusal of any reason stays final when only its exit fact reaches the start. ...(cause instanceof AgentSessionAcquisitionRefusal @@ -157,7 +159,7 @@ export function providerStartupFailureFact(cause?: unknown): SubmissionRejection : {}), detail: providerDiagnosticOf(cause), argumentProblem: argumentProblemOf(cause), - ...(exited ? {} : { requestTimeout: providerRequestTimeoutOf(cause) }) + requestTimeout }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts index 2680f14687e2..281add0eea6b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts @@ -15,6 +15,7 @@ import { HOST_TEST_SESSION, hostTestMessage } from './structured-agent-session-h import canonicalRefusal from '../../codex/app-server/fixtures/start-resume-canonical-filename-refusal.json' import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' +import { acpStartFailure } from '../../acp/acp-startup-outcome' import { agentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' import { structuredAgentSessionStartFailureRowIdentity } from '../../../shared/agent-session/failures/structured-agent-session-start-failure-row-key' @@ -111,3 +112,41 @@ it("records one standalone row in the start's words when a direct attach fails w expect(snapshot.submissions).toEqual([]) expect(dispatch).not.toHaveBeenCalled() }) + +it("keeps the exit's words for a direct attach whose exit settled before its final answer", async () => { + const { host, open, dispatch, pending } = awaitOpen() + await attach() + await vi.waitFor(() => expect(open).toHaveBeenCalledOnce()) + const child = host.collaboratorsForTests().sessions.get(HOST_TEST_SESSION)!.child! + await host.handleAdapterEvent({ + type: 'ended', + sessionId: HOST_TEST_SESSION, + fence: child.fence, + acquisitionGeneration: child.generation!, + cause: 'unexpected-exit', + reason: 'opencode exited', + failure: { kind: 'providerExited' } + }) + // The agent's sign-in answer drained after its exit; it stays final but words nothing new. + pending.reject( + acpStartFailure( + new AgentSessionAcquisitionRefusal('OpenCode requires sign-in', 'notSignedIn'), + true + ) + ) + await vi.waitFor(async () => { + const rows = (await host.journalSnapshot(HOST_TEST_SESSION)).items.filter( + (item) => item.body.kind === 'status' + ) + expect(rows).toEqual([ + expect.objectContaining({ + itemId: agentJournalItemKey(structuredAgentSessionStartFailureRowIdentity('awaited-open')), + body: expect.objectContaining({ + failure: expect.objectContaining({ kind: 'providerStartFailed' }) + }) + }) + ]) + }) + await vi.waitFor(() => expect(open).toHaveBeenCalledOnce()) + expect(dispatch).not.toHaveBeenCalled() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts index 4a9f11d42e54..3e9645f94fc3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts @@ -31,7 +31,14 @@ export async function openStructuredAgentSessionProviderChild( child: StructuredAgentSessionProviderChildIdentity ): Promise { const { startupAttempts, acquireAborts } = context.runtimeState + // Once this owner is done, nothing reads how the acquisition ended before ready. + const retire = (): void => { + if (child.generation !== null) { + context.deps.adapter.retireStartup?.(sessionId, child.generation) + } + } if (!startupAttempts.own(sessionId, child)) { + retire() return } const attempt = acquireAborts.begin(sessionId) @@ -71,6 +78,7 @@ export async function openStructuredAgentSessionProviderChild( } finally { startupAttempts.disown(sessionId, child) attempt.end() + retire() } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts index cd8059ea1559..272f8b894fab 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts @@ -69,18 +69,30 @@ export async function settleStructuredAgentSessionProviderStarted( // Only the adapter's word on this exact acquisition says it died; absence alone says nothing. const ended = () => context.deps.adapter.startupEnding?.(event.sessionId, event.acquisitionGeneration) - // A death the adapter reports ends the start without waiting on its bookkeeping below. - const died = context.runtimeState.startupAttempts - .rootExit(event.sessionId, child) - .then(() => ended() ?? new Promise(() => {})) - return Promise.race([died, settleReady(context, event, rewind, ended, signal)]) + const local: LocalSettlement = { started: false, fenced: false } + // A death the adapter reports ends the start without waiting on its bookkeeping below, but + // never once Orca's own handle or rewind write began: only that write's answer says whether + // the start may run again, and the contract guard bounds the wait for it. + const died = context.runtimeState.startupAttempts.rootExit(event.sessionId, child).then(() => { + const ending = local.started ? undefined : ended() + if (!ending) { + return new Promise(() => {}) + } + local.fenced = true + return ending + }) + return Promise.race([died, settleReady(context, event, rewind, ended, local, signal)]) } +/** `fenced`: the death decided the start, so its local writes never begin. */ +type LocalSettlement = { started: boolean; fenced: boolean } + function settleReady( context: StructuredAgentSessionProviderStartedContext, event: OpenedProvider, rewind: StructuredRewindRecoveryEvidence, ended: () => AgentSessionStartFailure | undefined, + local: LocalSettlement, signal?: AbortSignal ): Promise { const child = { generation: event.acquisitionGeneration, fence: event.fence } @@ -102,15 +114,22 @@ function settleReady( ) { return 'stale' } - if (ending) { - return ending + if (ending || local.fenced) { + return ending ?? 'stale' } + local.started = true if ( !(await recordStartedHandle(context, event)) || !(await applyRewindAtStart(context, event.sessionId, current, rewind)) ) { return refusedStart() } + signal?.throwIfAborted() + // The writes succeeded; a death meanwhile is still this start's outcome. + const endedWhileWriting = ended() + if (endedWhileWriting) { + return endedWhileWriting + } if ( !context.runtimeState.startupAttempts.onClock(event.sessionId, child) || !markProviderChildStarted(current, child) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts index 060cb79332db..cb74e7ba38f2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts @@ -51,109 +51,146 @@ export function registerStructuredAgentSessionStart( signal: acquire.signal, children }) - const outcome = context.tasks.trackAttach( - Promise.resolve() - .then(async (): Promise => { - let result: StructuredAgentSessionResumeOutcome = failedStructuredAgentSessionStart() - for (let number = 0; number < 5; number++) { + const attempting = Promise.resolve() + .then(async (): Promise => { + let result: StructuredAgentSessionResumeOutcome = failedStructuredAgentSessionStart() + for (let number = 0; number < 5; number++) { + throwIfSignalAborted(acquire.signal) + if (attempts.pastCeiling(registeredAt)) { + result = { ...failedStructuredAgentSessionStart(), expired: true } + break + } + const lock = structuredAgentSessionStartFlightLock({ + sessionId, + attemptId, + signal: acquire.signal, + flights, + serialize: context.serialize + }) + const stillQueued = await lock.commit(async () => + structuredAgentSessionStartStillQueued(context.sessions.get(sessionId), startedFor) + ) + if (!stillQueued) { + return { ...failedStructuredAgentSessionStart(), aborted: true } + } + result = await run(lock, attemptId) + if (result.ok) { + // The child's open owns its outcome; readiness ends once that open gave it. + const ready = await lock.wait(handle.ended) throwIfSignalAborted(acquire.signal) - if (attempts.pastCeiling(registeredAt)) { - result = { ...failedStructuredAgentSessionStart(), expired: true } - break + if (ready === 'ready') { + return result } - const lock = structuredAgentSessionStartFlightLock({ - sessionId, - attemptId, - signal: acquire.signal, - flights, - serialize: context.serialize - }) - const stillQueued = await lock.commit(async () => - structuredAgentSessionStartStillQueued(context.sessions.get(sessionId), startedFor) + const owned = handle.result + result = await lock.commit(async () => + structuredAgentSessionOpenFailure(context.sessions.get(sessionId), owned) ) - if (!stillQueued) { - return { ...failedStructuredAgentSessionStart(), aborted: true } - } - result = await run(lock, attemptId) - if (result.ok) { - // The child's open owns its outcome; readiness ends once that open gave it. - const ready = await lock.wait(handle.ended) - throwIfSignalAborted(acquire.signal) - if (ready === 'ready') { - return result - } - const owned = handle.result - result = await lock.commit(async () => - structuredAgentSessionOpenFailure(context.sessions.get(sessionId), owned) - ) - } - if (result.ok || result.aborted || result.expired || !result.retry || number === 4) { - break - } - attempts.abandon(sessionId, attemptId) - await waitForPromiseWithSignal(delay(100 * 2 ** number), acquire.signal) - throwIfSignalAborted(acquire.signal) - attemptId = randomUUID() - handle = attempts.begin(sessionId, attemptId, registeredAt) - register() } - return result - }) - .catch((error: unknown): StructuredAgentSessionResumeOutcome => { - if (!acquire.signal.aborted) { - context.deps.logger.warn('starting the agent failed', { - scope: 'agent-start', - sessionId, - error - }) - return { - ...failedStructuredAgentSessionStart(), - failure: agentSessionFailureFact('hostFault') - } - } - return failedStructuredAgentSessionStart() - }) - .then(async (result): Promise => { - if (result.ok) { - return result + if (result.ok || result.aborted || result.expired || !result.retry || number === 4) { + break } - return context.serialize(sessionId, async () => { - const settled = acquire.signal.aborted - ? stoppedStructuredAgentSessionStart(acquire.signal.reason) - : result - if (!settled.aborted && (startedFor !== undefined || children.length > 0)) { - const session = context.sessions.get(sessionId) - if (session) { - const record = context.deps.store.getRecord(sessionId) - const failure = failedStart(record, settled) - await recordStructuredAgentSessionStartFailure( - { journal: session.journal, fence: record?.lease.runtimeFence ?? 0 }, - { - startKey: - children.at(-1)?.generation ?? failure.startKey ?? startedFor ?? attemptId, - ...structuredAgentSessionStartFailure( - failure.cause, - structuredAgentSessionFailureWordsContext(record) - ) - }, - startedFor ?? null - ) - } - } - return settled - }) - }) - .finally(() => { - acquire.end() - flights.end(sessionId, attemptId) attempts.abandon(sessionId, attemptId) - context.publishStatus(sessionId) - }) - ) + await waitForPromiseWithSignal(delay(100 * 2 ** number), acquire.signal) + throwIfSignalAborted(acquire.signal) + attemptId = randomUUID() + handle = attempts.begin(sessionId, attemptId, registeredAt) + register() + } + return result + }) + .catch((error: unknown): StructuredAgentSessionResumeOutcome => { + if (!acquire.signal.aborted) { + context.deps.logger.warn('starting the agent failed', { + scope: 'agent-start', + sessionId, + error + }) + return { + ...failedStructuredAgentSessionStart(), + failure: agentSessionFailureFact('hostFault') + } + } + return failedStructuredAgentSessionStart() + }) + let recorded: Promise = Promise.resolve() + // Stop, close, quit, the deadline and the contract guard decide the start at once, even while + // its attempt or the session's serialized work is still held; late attempt work fails the + // flight's commit check. The failure row is queued before callers wake, so their work follows + // it, but they never wait for it to be written. + const outcome = Promise.race([ + attempting, + stoppedStructuredAgentSessionStartSignal(acquire.signal) + ]) + .then((result): StructuredAgentSessionResumeOutcome => { + const settled = + result.ok || !acquire.signal.aborted + ? result + : stoppedStructuredAgentSessionStart(acquire.signal.reason) + if (!settled.ok && !settled.aborted && (startedFor !== undefined || children.length > 0)) { + const key = { generation: children.at(-1)?.generation, startedFor, attemptId } + recorded = context + .serialize(sessionId, () => + recordStructuredAgentSessionFlightFailure(context, sessionId, settled, key) + ) + .catch((error: unknown) => + context.deps.logger.warn('recording a failed start did not finish', { + scope: 'agent-start', + sessionId, + error + }) + ) + } + return settled + }) + .finally(() => { + acquire.end() + flights.end(sessionId, attemptId) + attempts.abandon(sessionId, attemptId) + context.publishStatus(sessionId) + }) + context.tasks.trackAttach(outcome.then(() => Promise.all([attempting, recorded]))) register() return outcome } +function stoppedStructuredAgentSessionStartSignal( + signal: AbortSignal +): Promise { + return new Promise((resolve) => { + const stopped = (): void => resolve(stoppedStructuredAgentSessionStart(signal.reason)) + if (signal.aborted) { + stopped() + } else { + signal.addEventListener('abort', stopped, { once: true }) + } + }) +} + +async function recordStructuredAgentSessionFlightFailure( + context: StructuredAgentSessionAttachContext, + sessionId: string, + settled: Extract, + key: { generation: string | null | undefined; startedFor: string | undefined; attemptId: string } +): Promise { + const session = context.sessions.get(sessionId) + if (!session) { + return + } + const record = context.deps.store.getRecord(sessionId) + const failure = failedStart(record, settled) + await recordStructuredAgentSessionStartFailure( + { journal: session.journal, fence: record?.lease.runtimeFence ?? 0 }, + { + startKey: key.generation ?? failure.startKey ?? key.startedFor ?? key.attemptId, + ...structuredAgentSessionStartFailure( + failure.cause, + structuredAgentSessionFailureWordsContext(record) + ) + }, + key.startedFor ?? null + ) +} + export function failedStructuredAgentSessionStart(): Extract< StructuredAgentSessionResumeOutcome, { ok: false } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts index 8e0708440882..719ddda62ac9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts @@ -107,18 +107,34 @@ export function agentSessionAcquireFailure( return agentSessionStartVerdict(error, answer) ?? error } -/** Each session's latest acquisition that ended before the host committed it ready, with the - * adapter's verdict; the session's next acquisition or the adapter's close disposes it. */ +/** How each starting acquisition ended before the host committed it ready, with the adapter's + * verdict. An acquisition is admitted when it starts and retired once nothing can read its ending: + * its start's owner finished, it failed, or its session was released, disposed or closed. A + * retired acquisition records nothing, so an exit after ready never leaves an entry behind. */ export class AgentSessionStartEndings { private readonly endings = new Map< string, - { generation: string; failure: AgentSessionStartFailure } + { generation: string; failure?: AgentSessionStartFailure } >() - /** The first ending of an acquisition is its verdict; later reports of it change nothing. */ + /** The session's new acquisition; an earlier one's ending can no longer be read. */ + admit(sessionId: string, generation: string): void { + this.endings.set(sessionId, { generation }) + } + + /** Admits the acquisition an acquire returned, for adapters that record only published ones. */ + admitting(sessionId: string) { + return (acquisition: T): T => { + this.admit(sessionId, acquisition.acquisitionGeneration) + return acquisition + } + } + + /** The first ending of an admitted acquisition is its verdict; later reports change nothing. */ record(sessionId: string, generation: string, failure: AgentSessionStartFailure): void { - if (this.endings.get(sessionId)?.generation !== generation) { - this.endings.set(sessionId, { generation, failure }) + const admitted = this.endings.get(sessionId) + if (admitted?.generation === generation) { + admitted.failure ??= failure } } @@ -127,8 +143,11 @@ export class AgentSessionStartEndings { return ended?.generation === generation ? ended.failure : undefined } - forget(sessionId: string): void { - this.endings.delete(sessionId) + /** Without a generation, retires whatever the session holds. */ + retire(sessionId: string, generation?: string): void { + if (generation === undefined || this.endings.get(sessionId)?.generation === generation) { + this.endings.delete(sessionId) + } } clear(): void { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts index 54e3e0dff6c8..602ec51cc9de 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts @@ -679,12 +679,7 @@ describe('one logical start across physical attempts', () => { const expectedMessages = [later] expect(await submission(later)).toMatchObject({ dispatchState: 'pending' }) if (failsWrite) { - await eventually(() => - expect(host.collaboratorsForTests().conversationDelivery.loop.isRunning(SESSION)).toBe( - false - ) - ) - expect(await submission(later)).toMatchObject({ dispatchState: 'pending' }) + // The failed start's row not being written holds up no later message. restoreWrite?.() expectedMessages.push(await accept('after the journal recovered')) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts new file mode 100644 index 000000000000..f594acd15836 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts @@ -0,0 +1,182 @@ +import { PassThrough } from 'node:stream' +import { describe, expect, it, vi } from 'vitest' +import { createAcpAgentConnection } from '../../acp/acp-agent-connection' +import { openAcpAdapterRig } from '../../acp/acp-structured-adapter.test-support' +import { readAntigravitySignInLine } from '../../antigravity/antigravity-sign-in-url' +import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' +import { adapterFor, fakeCodex } from '../../codex/codex-structured-session-adapter-fixture' +import { createFakePipedChild } from '../../../shared/__fixtures__/fake-spawned-child' +import { closeProviderTimelineRigs } from '../agent-session-timeline/provider-timeline-assembler-test-support' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { recordStartedHandle } from './structured-agent-session-start-provider-answer' +import type * as ProviderAnswer from './structured-agent-session-start-provider-answer' +import { STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS } from './structured-agent-session-startup-attempt-contract' +import { + accept, + acquire, + dispatch, + eventually, + heldBehindStart, + host, + observeRootExit, + proveOpen, + providerOpen, + publishFirst, + recordStartupEnding, + registerStartupAttemptHostTests, + restartWith, + submission +} from './structured-agent-session-startup-attempt-host-test-support' + +vi.mock('./structured-agent-session-start-provider-answer', async (original) => { + const actual = await original() + return { ...actual, recordStartedHandle: vi.fn(actual.recordStartedHandle) } +}) + +registerStartupAttemptHostTests() + +/** Holds the session's serialized work, as a stalled journal write would. */ +async function holdSerializedWork(): Promise<() => Promise> { + const held = Promise.withResolvers() + const entered = Promise.withResolvers() + const blocked = host.collaboratorsForTests().serialize(SESSION, async () => { + entered.resolve() + await held.promise + }) + await entered.promise + return async () => { + held.resolve() + await blocked + } +} + +describe('a start that cannot finish', () => { + it.each([ + ['its process exited and the adapter never answered', 'providerStartFailed', 0], + ['it outlived its ceiling', 'expired', 20_000] + ] as const)( + 'releases its callers when %s, while the session is busy, and records its row once after', + async (_case, decided, waitMs) => { + await restartWith(publishFirst, { startupLimits: { ceilingMs: 20_000 } }) + providerOpen.mockImplementationOnce(() => new Promise(() => {})) + const id = await heldBehindStart('one request') + const flight = host.collaboratorsForTests().runtimeState.startFlights.get(SESSION)! + const release = await holdSerializedWork() + let outcome: unknown + void flight.outcome.then((settled) => { + outcome = settled + }) + if (waitMs === 0) { + observeRootExit() + } + await vi.advanceTimersByTimeAsync(waitMs || STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS) + expect(outcome).toMatchObject( + decided === 'expired' + ? { ok: false, expired: true } + : { ok: false, failure: { kind: 'providerStartFailed' } } + ) + await release() + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + const rows = (await host.journalSnapshot(SESSION)).items.filter( + (item) => item.body.kind === 'status' + ) + expect(rows).toHaveLength(1) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + } + ) + + it("wakes on Codex's own exit while spawn bookkeeping is still held, and never readies after", async () => { + const codex = fakeCodex() + const identity = Promise.withResolvers() + const adapter = adapterFor(codex, {}, [], { readProcessStartTime: () => identity.promise }) + await restartWith((input) => adapter.acquire(input)) + const id = await accept('a start whose process dies before its identity is saved') + await eventually(() => expect(codex.connections).toHaveLength(1)) + const child = codex.connections[0] + child.closed = true + child.handlers.onExit?.(new CodexAppServerExitError('codex app-server connection ended'), { + expected: false + }) + await vi.advanceTimersByTimeAsync(STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS) + expect(host.collaboratorsForTests().runtimeState.startupAttempts.isOpen(SESSION)).toBe(false) + expect(acquire.mock.calls[0][0].signal?.aborted).toBe(true) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + identity.resolve(1_700_000_000_000) + await vi.advanceTimersByTimeAsync(1_000) + expect(host.readStatusSummary(SESSION)?.hostExecutionPhase).not.toBe('ready') + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + await adapter.closeAll() + }) +}) + +describe("Orca's own refusal during a start", () => { + it('stays final when its handle write refuses after the agent died unanswered', async () => { + await restartWith(publishFirst) + const write = Promise.withResolvers() + const entered = Promise.withResolvers() + vi.mocked(recordStartedHandle).mockImplementationOnce(() => { + entered.resolve() + return write.promise + }) + const id = await heldBehindStart('the thread opened; saving its handle is still pending') + await proveOpen() + await entered.promise + recordStartupEnding() + observeRootExit() + await vi.advanceTimersByTimeAsync(10) + write.resolve(false) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect((await submission(id))?.rejection).toMatchObject({ kind: 'hostStopped' }) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('stays final when an ACP agent prints a sign-in line Orca cannot read before it initializes', async () => { + const connections: ReturnType[] = [] + const children: ReturnType[] = [] + const rig = await openAcpAdapterRig({ + deps: { + connect: (launch, options) => { + const child = Object.assign(createFakePipedChild(), { + pid: 9_999_999, + stdout: new PassThrough(), + stdin: new PassThrough(), + stderr: new PassThrough(), + kill: () => true + }) + children.push(child) + // Orca's cleanup ends stdin; the agent then exits. + child.stdin.once('finish', () => child.emit('exit', 0, null)) + const connection = createAcpAgentConnection( + launch, + { ...options, signIn: { read: readAntigravitySignInLine } }, + () => child + ) + connections.push(connection) + return connection + }, + // The refusal and its cleanup exit land while the spawn's identity is still being saved. + readProcessStartTime: async () => { + children + .at(-1)! + .stdout.write('Open the following link to authenticate the ACP server: not-a-url\n') + await eventually(() => expect(connections.at(-1)!.exited).toBe(true)) + return 1_700_000_000_000 + } + } + }) + try { + await restartWith((input) => rig.adapter.acquire(input)) + const id = await accept('an agent whose sign-in line Orca cannot read') + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect(connections).toHaveLength(1) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + } finally { + await rig.adapter.closeAll() + await closeProviderTimelineRigs() + } + }) +}) diff --git a/src/main/pi/rpc-session-adapter.ts b/src/main/pi/rpc-session-adapter.ts index 78030cab9981..c55f9f5dbac8 100644 --- a/src/main/pi/rpc-session-adapter.ts +++ b/src/main/pi/rpc-session-adapter.ts @@ -110,6 +110,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { ? error : new AgentSessionPreSpawnError(error) } + this.endings.retire(id, session.generation) const result = await session.close(false).catch(() => null) const exited = piStartExitError(error, session.connection, result) if (exited) { @@ -228,12 +229,15 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { holdsDispatch = (id: string): boolean => this.sessions.get(id)?.turns.holdsDispatch ?? false heldSends = (id: string): ReadonlySet | undefined => this.sessions.get(id)?.heldSends startupEnding = (id: string, generation: string) => this.endings.get(id, generation) + retireStartup = (id: string, generation: string) => this.endings.retire(id, generation) holdsLiveProviderProcess(id: string, generation: string): boolean { const session = this.sessions.get(id) return session?.generation === generation && !session.connection.closed } closeSession(id: string): Promise { + // A released or closed session's start has no reader left for its ending. + this.endings.retire(id) return this.close(id, true) } private async close(id: string, requested: boolean): Promise { diff --git a/src/main/pi/rpc-startup-open.ts b/src/main/pi/rpc-startup-open.ts index 99dde72ef392..b3bdbf420437 100644 --- a/src/main/pi/rpc-startup-open.ts +++ b/src/main/pi/rpc-startup-open.ts @@ -16,7 +16,7 @@ export function watchPiStartExit( endings: AgentSessionStartEndings ): void { const sessionId = session.input.identity.sessionId - endings.forget(sessionId) + endings.admit(sessionId, session.generation) session.connection.onExit((requested) => { if (!requested) { input.events?.observeRootExit?.() From a8e3da90df82b529a4b34ebca803d72a74034266 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:37:19 -0800 Subject: [PATCH 16/21] fix(agent-session): let a start's owner, not its live route, keep and retire its ending Exit acknowledgment drops a session's route while the start's owner may still be reading how its acquisition ended, so the router now reaches the adapters by exact generation for both the read and the retirement. Claude admits its acquisition before publication can observe an exit. An ACP start Orca refused before the agent exited now fails with that original refusal as the adapter's final verdict, through every cleanup wrapper, instead of a generic closed-while-starting error. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/main/acp/acp-agent-connection.ts | 18 +++++- src/main/acp/acp-startup-outcome.ts | 9 ++- .../acp/acp-structured-child-lifecycle.ts | 8 ++- src/main/acp/acp-structured-connection.ts | 5 +- .../acp/acp-structured-session-adapter.ts | 29 +++++----- .../claude-structured-session-acquisition.ts | 2 +- .../claude-structured-session-adapter.ts | 13 +++-- .../claude-structured-session-state.ts | 2 + .../claude-structured-start-ending.ts | 22 +++++++ .../codex-structured-start-endings.test.ts | 58 +++++++++++++++++++ ...structured-agent-session-adapter-router.ts | 17 +++--- .../structured-agent-session-start-failure.ts | 8 --- ...red-agent-session-startup-liveness.test.ts | 27 ++++++++- 13 files changed, 172 insertions(+), 46 deletions(-) diff --git a/src/main/acp/acp-agent-connection.ts b/src/main/acp/acp-agent-connection.ts index 0e1ff1561901..2a24ebee24eb 100644 --- a/src/main/acp/acp-agent-connection.ts +++ b/src/main/acp/acp-agent-connection.ts @@ -51,7 +51,8 @@ type AcpConnectionLifecycle = { error?: Error /** The process's own failure, such as its spawn; it is no refusal by Orca. */ processError?: Error - /** Why the connection closed before the process exited, unless the process itself failed. */ + /** Why the connection closed before the process exited, unless Orca closed it or the process + * itself failed. Taken when it closed. */ lost?: Error finishExit?: () => void outputEndTimer?: ReturnType @@ -121,6 +122,14 @@ export class AcpAgentConnection extends AcpSessionRuntime { lifecycle.outputEndTimer.unref() }, onClose: (error) => { + if ( + !lifecycle.error && + !lifecycle.closing && + managed.rootVerdict !== 'exited' && + error !== lifecycle.processError + ) { + lifecycle.lost = error + } lifecycle.error ??= error if (lifecycle.closing || managed.rootVerdict === 'exited') { return @@ -166,7 +175,6 @@ export class AcpAgentConnection extends AcpSessionRuntime { super.close(new AcpConnectionClosedError('The agent printed a sign-in link Orca cannot read')) child.stderr.on('error', (error) => super.close(error)) managed.onExit((exit) => { - lifecycle.lost = lifecycle.error === lifecycle.processError ? undefined : lifecycle.error const expected = lifecycle.closing const finish = (): void => { clearTimeout(lifecycle.outputEndTimer) @@ -233,6 +241,12 @@ export class AcpAgentConnection extends AcpSessionRuntime { return this.managed.exitStatus } + /** Why the connection closed before the process exited, such as Orca refusing what the agent + * wrote; never the process's own failure or its exit. Kept from the moment it closed. */ + get lostBeforeExit(): Error | undefined { + return this.lifecycle.lost + } + /** `requested`: Orca was closing the agent when it exited. `lost`: why the connection had * already closed before the exit, such as Orca refusing what the agent wrote. */ onExit(listener: (exit: ProviderProcessExit, requested?: boolean, lost?: Error) => void): void { diff --git a/src/main/acp/acp-startup-outcome.ts b/src/main/acp/acp-startup-outcome.ts index aa6e1118b5f4..b4076070a2df 100644 --- a/src/main/acp/acp-startup-outcome.ts +++ b/src/main/acp/acp-startup-outcome.ts @@ -1,8 +1,8 @@ import { agentSessionAcquireFailure, agentSessionStartFailure, - type AgentSessionStartAnswer, - type AgentSessionStartFailure + AgentSessionStartFailure, + type AgentSessionStartAnswer } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' import { AcpAgentError, @@ -39,6 +39,11 @@ export function acpStartFailure(error: unknown, exited = false): AgentSessionSta return agentSessionStartFailure(error, acpStartupAnswer, exited) } +/** Orca closed the connection before the agent exited, refusing what it wrote: final. */ +export function acpRefusedStart(lost: Error): AgentSessionStartFailure { + return new AgentSessionStartFailure(lost, 'never') +} + export function acpAcquireFailure(error: unknown): unknown { return agentSessionAcquireFailure(error, acpStartupAnswer) } diff --git a/src/main/acp/acp-structured-child-lifecycle.ts b/src/main/acp/acp-structured-child-lifecycle.ts index 9c705930bd5a..35a4359aa791 100644 --- a/src/main/acp/acp-structured-child-lifecycle.ts +++ b/src/main/acp/acp-structured-child-lifecycle.ts @@ -24,7 +24,7 @@ import { import { windDownAcpTurn } from './acp-structured-stop' import { AcpAgentExitedError } from './acp-errors' import { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' -import { acpStartFailure } from './acp-startup-outcome' +import { acpRefusedStart, acpStartFailure } from './acp-startup-outcome' /** An agent that never ran names its missing CLI or folder; any other end while starting is its exit. */ function acpStartupEndFailure(session: AcpStructuredChild): SubmissionRejectionFact { @@ -203,7 +203,11 @@ export class AcpStructuredChildren { lost ?? new AcpAgentExitedError(detail, connection.stderrTail()), connection.spawnAbsence ) - this.endings.record(sessionId, generation, acpStartFailure(exited)) + this.endings.record( + sessionId, + generation, + lost ? acpRefusedStart(exited) : acpStartFailure(exited) + ) } private recordEnding(session: AcpStructuredChild, error: Error): void { diff --git a/src/main/acp/acp-structured-connection.ts b/src/main/acp/acp-structured-connection.ts index a4a3ded7c6a1..630b4ca9a870 100644 --- a/src/main/acp/acp-structured-connection.ts +++ b/src/main/acp/acp-structured-connection.ts @@ -24,7 +24,10 @@ export type AcpStructuredConnection = Pick< | 'resumeReading' | 'close' | 'processTreeUnproven' -> & { readonly lastCloseResult?: AcpAgentConnection['lastCloseResult'] } +> & { + readonly lastCloseResult?: AcpAgentConnection['lastCloseResult'] + readonly lostBeforeExit?: AcpAgentConnection['lostBeforeExit'] +} export type ConnectAcpAgent = ( launch: ProviderProcessLaunch, diff --git a/src/main/acp/acp-structured-session-adapter.ts b/src/main/acp/acp-structured-session-adapter.ts index 9ac78a39244f..b8ebae7203d9 100644 --- a/src/main/acp/acp-structured-session-adapter.ts +++ b/src/main/acp/acp-structured-session-adapter.ts @@ -37,7 +37,7 @@ import { AcpStructuredChildren } from './acp-structured-child-lifecycle' import { AcpConversationCopies } from './acp-conversation-copies' import { AcpTextTasks } from './acp-text-task' import { supportsAcpStructuredLocation } from './acp-structured-location-support' -import { acpAcquireFailure } from './acp-startup-outcome' +import { acpAcquireFailure, acpRefusedStart } from './acp-startup-outcome' import { openAcpStart } from './acp-structured-start-open' export class AcpStructuredSessionAdapter implements SessionAdapter { @@ -91,9 +91,6 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { const sessionId = input.identity.sessionId const generation = this.deps.mintGeneration?.() ?? randomUUID() this.children.endings.admit(sessionId, generation) - // Orca closing the connection first, such as on a sign-in line it cannot read, is the start's - // cause; the exit its cleanup then causes is not the agent ending unanswered. - let lostFirst: Error | undefined try { const { acquisition, session } = await acquireAcpStructuredSession({ acquire: input, @@ -102,7 +99,6 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { abandoned: () => attempt.signal.aborted, track: (connection) => { connection.onExit((_exit, requested, lost) => { - lostFirst ??= lost if (!requested) { input.events?.observeRootExit?.() this.children.recordExit(sessionId, generation, connection, lost) @@ -133,15 +129,22 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { // Wait briefly for process exit so startup failures include final diagnostics. await waitForAcpExit(connection, this.stopGraceMs(), attempt.signal) } - // Capture spontaneous exit before close can classify it as requested. - const exitedOnItsOwn = connection?.exited === true && !attempt.signal.aborted && !lostFirst - if (connection && !(await connection.close().catch(() => false))) { + // Orca closing the connection first, such as on a sign-in line it cannot read, is the start's + // cause; the exit its cleanup then causes is not the agent ending unanswered. Capture + // spontaneous exit before close can classify it as requested. + const exitedOnItsOwn = + connection?.exited === true && !attempt.signal.aborted && !connection.lostBeforeExit + const cleaned = connection && (await connection.close().catch(() => false)) + // Read after cleanup, which may be what observes the refusal; it outranks generic errors. + const refused = connection?.lostBeforeExit + const cause = refused ? acpRefusedStart(refused) : error + if (connection && !cleaned) { // Retain failed cleanup for retry before another child can spawn. this.children.starts.retainFailed(sessionId, connection) - throw new AgentSessionAcquisitionExitUnprovenError(error) + throw new AgentSessionAcquisitionExitUnprovenError(cause) } if (connection?.lastCloseResult?.killDelivered && !connection.exited) { - throw new AgentSessionAcquisitionRootExitObservedError(error, 'kill-delivered') + throw new AgentSessionAcquisitionRootExitObservedError(cause, 'kill-delivered') } if (attempt.signal.aborted) { const closed = new Error( @@ -151,8 +154,8 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { throw connection ? closed : new AgentSessionPreSpawnError(closed) } // Once the exit is proven, the start's error names a CLI or folder the spawn found missing. - if (error instanceof Error && connection?.exited) { - withProviderSpawnAbsence(error, connection.spawnAbsence) + if (cause instanceof Error && connection?.exited) { + withProviderSpawnAbsence(cause, connection.spawnAbsence) } if (connection && exitedOnItsOwn && !isAgentSessionPreSpawnError(error)) { throw new AgentSessionAcquisitionExitProvenError( @@ -161,7 +164,7 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { ) ) } - throw error + throw cause } } diff --git a/src/main/claude/structured/claude-structured-session-acquisition.ts b/src/main/claude/structured/claude-structured-session-acquisition.ts index e782573d07b9..ec77bf36e4c8 100644 --- a/src/main/claude/structured/claude-structured-session-acquisition.ts +++ b/src/main/claude/structured/claude-structured-session-acquisition.ts @@ -283,7 +283,7 @@ export async function acquireClaudeSession({ }) acquisitions.deleteIfCurrent(sessionId, attempt) await withAgentSessionCreatePhase('publish', input.recordPhase, async () => { - sessions.set(sessionId, session) + callbacks.publish(session) attempt.published = true for (const event of attempt.buffered.splice(0)) { event() diff --git a/src/main/claude/structured/claude-structured-session-adapter.ts b/src/main/claude/structured/claude-structured-session-adapter.ts index 26b7bf1fd543..dd9db92aa341 100644 --- a/src/main/claude/structured/claude-structured-session-adapter.ts +++ b/src/main/claude/structured/claude-structured-session-adapter.ts @@ -1,7 +1,7 @@ import { dispatchClaudeCommand } from './claude-structured-command-dispatch' import { AgentSessionAcquisitionRootExitObservedError } from '../../native-chat/agent-session-wire/structured-agent-session-acquisition-errors' import { claudeAcquireFailure } from '../claude-startup-failure' -import { openClaudeStart } from './claude-structured-start-ending' +import { claudeStartPublication, openClaudeStart } from './claude-structured-start-ending' import { AgentSessionStartEndings } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' import type { StructuredAgentSessionAcquireInput, @@ -77,6 +77,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda this.endings.retire(input.identity.sessionId) const record = this.deps.namingRecord?.(input.identity.sessionId) const naming = this.naming?.acquire(input, record) + const start = claudeStartPublication(this.sessions, this.endings, input.identity.sessionId) return acquireClaudeSession({ input, deps: this.deps, @@ -85,6 +86,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda exits: this.exits, ...(naming ? { naming } : {}), callbacks: { + publish: start.publish, deliver: (attempt, sessionId, event) => this.deliver(attempt, sessionId, event), emit: (session, _events, event) => this.emit(session, event), handleExit: (sessionId, attempt, error) => @@ -100,12 +102,11 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda settleExit: (sessionId, exit) => settleClaudeUnexpectedExit(this.exitLifecycle, sessionId, exit) } + }).catch((error: unknown) => { + start.failed() + naming?.failed() + throw claudeAcquireFailure(error) }) - .then(this.endings.admitting(input.identity.sessionId)) - .catch((error: unknown) => { - naming?.failed() - throw claudeAcquireFailure(error) - }) } private deliver(attempt: ClaudeAcquisitionAttempt, sessionId: string, event: () => void): void { diff --git a/src/main/claude/structured/claude-structured-session-state.ts b/src/main/claude/structured/claude-structured-session-state.ts index 850b3d87aba9..9479e5115dae 100644 --- a/src/main/claude/structured/claude-structured-session-state.ts +++ b/src/main/claude/structured/claude-structured-session-state.ts @@ -358,6 +358,8 @@ export async function cancelClaudeAcquisitionAttempt( } export type ClaudeAcquireCallbacks = { + /** Publishes the session, its start ending admitted first so its exit is never missed. */ + publish: (session: ClaudeSession) => void deliver: (attempt: ClaudeAcquisitionAttempt, sessionId: string, event: () => void) => void emit: ( session: ClaudeSession | null, diff --git a/src/main/claude/structured/claude-structured-start-ending.ts b/src/main/claude/structured/claude-structured-start-ending.ts index 99be5dc066b5..a685b52f6625 100644 --- a/src/main/claude/structured/claude-structured-start-ending.ts +++ b/src/main/claude/structured/claude-structured-start-ending.ts @@ -12,6 +12,28 @@ import { openClaudeStructuredSession } from './claude-structured-session-open' import { claudeStartFailure } from '../claude-startup-failure' import type { ClaudeSession, ClaudeSessionExit } from './claude-structured-session-state' +/** Publishes an acquisition's session with its start ending admitted first, so its exit is never + * missed. A failed acquire has no start owner left, so it retires exactly that admission. */ +export function claudeStartPublication( + sessions: Map, + endings: AgentSessionStartEndings, + sessionId: string +): { publish: (session: ClaudeSession) => void; failed: () => void } { + let admitted: string | undefined + return { + publish: (session) => { + admitted = session.acquisitionGeneration + endings.admit(sessionId, admitted) + sessions.set(sessionId, session) + }, + failed: () => { + if (admitted !== undefined) { + endings.retire(sessionId, admitted) + } + } + } +} + /** Claude ended on its own: unanswered, unless what ended it was its answer or a refusal. */ export function claudeExitEnding(error: Error): AgentSessionStartFailure { return claudeStartFailure(withObservedProviderExit(new Error(error.message, { cause: error }))) diff --git a/src/main/codex/codex-structured-start-endings.test.ts b/src/main/codex/codex-structured-start-endings.test.ts index 5a55b59fb687..77747036e22e 100644 --- a/src/main/codex/codex-structured-start-endings.test.ts +++ b/src/main/codex/codex-structured-start-endings.test.ts @@ -1,5 +1,6 @@ import { expect, it } from 'vitest' import { CodexAppServerExitError } from './app-server/codex-app-server-exit-error' +import { claudeAndCodexRouter } from '../native-chat/agent-session-wire/structured-agent-session-adapter-router-test-support' import { adapterFor, fakeCodex, identityFor } from './codex-structured-session-adapter-fixture' it("keeps a start's ending only while that start can still read it", async () => { @@ -43,3 +44,60 @@ it("keeps a start's ending only while that start can still read it", async () => expect(retained([ready])).toEqual([]) await adapter.closeAll() }) + +it("keeps a routed start's ending until its owner retires it, after exit dropped the route", async () => { + const codex = fakeCodex() + const adapter = adapterFor(codex) + const router = claudeAndCodexRouter({ claude: adapterFor(fakeCodex()), codex: adapter }, () => + adapter.closeAll() + ) + const exitLatest = () => { + const connection = codex.connections.at(-1)! + connection.closed = true + const exit = () => + connection.handlers.onExit?.( + new CodexAppServerExitError('codex app-server connection ended'), + { + expected: false + } + ) + exit() + return exit + } + const starts: { id: string; generation: string }[] = [] + for (let n = 0; n < 10; n++) { + const id = `routed-${n}` + const acquired = await router.acquire({ + identity: identityFor(id), + fence: 1, + spawnToken: `s-${n}` + }) + exitLatest() + // The host acknowledges the exit before the start's owner has finished reading it. + router.acknowledgeSessionRelease(id) + expect(router.startupEnding(id, acquired.acquisitionGeneration)).toBeDefined() + router.retireStartup(id, acquired.acquisitionGeneration) + starts.push({ id, generation: acquired.acquisitionGeneration }) + } + expect(starts.filter(({ id, generation }) => adapter.startupEnding(id, generation))).toEqual([]) + + // An old owner's retirement leaves the replacement's ending, and a late old exit records nothing. + const old = await router.acquire({ + identity: identityFor('replaced'), + fence: 1, + spawnToken: 'r-1' + }) + const lateOldExit = exitLatest() + router.acknowledgeSessionRelease('replaced') + const next = await router.acquire({ + identity: identityFor('replaced'), + fence: 2, + spawnToken: 'r-2' + }) + exitLatest() + router.retireStartup('replaced', old.acquisitionGeneration) + expect(router.startupEnding('replaced', next.acquisitionGeneration)).toBeDefined() + lateOldExit() + expect(router.startupEnding('replaced', old.acquisitionGeneration)).toBeUndefined() + await adapter.closeAll() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts index b4398d46ff65..ef01e2bf3b9b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts @@ -156,16 +156,15 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi this.liveOwnerOrNull(sessionId)?.holdsLiveProviderProcess?.(sessionId, acquisitionGeneration) ?? false - // An ended child's route may already be stopped; its adapter still holds the ending. + // A start's owner outlives its route: exit acknowledgment may drop the route while the owner + // still reads its ending, so both calls reach the adapters by exact generation, not by route. startupEnding = (sessionId: string, acquisitionGeneration: string) => - this.routes - .get(sessionId) - ?.registration.adapter.startupEnding?.(sessionId, acquisitionGeneration) - - retireStartup = (sessionId: string, acquisitionGeneration: string) => - this.routes - .get(sessionId) - ?.registration.adapter.retireStartup?.(sessionId, acquisitionGeneration) + this.adapters() + .map((adapter) => adapter.startupEnding?.(sessionId, acquisitionGeneration)) + .find((ending) => ending !== undefined) + + retireStartup = (sessionId: string, acquisitionGeneration: string): void => + this.adapters().forEach((adapter) => adapter.retireStartup?.(sessionId, acquisitionGeneration)) awaitStoppedRequestEnd = async (sessionId: string, stoppedAt: number) => this.liveOwnerOrNull(sessionId)?.awaitStoppedRequestEnd?.(sessionId, stoppedAt) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts index 719ddda62ac9..ac3004b112ab 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts @@ -122,14 +122,6 @@ export class AgentSessionStartEndings { this.endings.set(sessionId, { generation }) } - /** Admits the acquisition an acquire returned, for adapters that record only published ones. */ - admitting(sessionId: string) { - return (acquisition: T): T => { - this.admit(sessionId, acquisition.acquisitionGeneration) - return acquisition - } - } - /** The first ending of an admitted acquisition is its verdict; later reports change nothing. */ record(sessionId: string, generation: string, failure: AgentSessionStartFailure): void { const admitted = this.endings.get(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts index f594acd15836..f0e5d3881986 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts @@ -8,6 +8,8 @@ import { adapterFor, fakeCodex } from '../../codex/codex-structured-session-adap import { createFakePipedChild } from '../../../shared/__fixtures__/fake-spawned-child' import { closeProviderTimelineRigs } from '../agent-session-timeline/provider-timeline-assembler-test-support' import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { findInErrorChain } from '../../provider-process/provider-error-chain' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' import { recordStartedHandle } from './structured-agent-session-start-provider-answer' import type * as ProviderAnswer from './structured-agent-session-start-provider-answer' import { STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS } from './structured-agent-session-startup-attempt-contract' @@ -136,6 +138,8 @@ describe("Orca's own refusal during a start", () => { it('stays final when an ACP agent prints a sign-in line Orca cannot read before it initializes', async () => { const connections: ReturnType[] = [] const children: ReturnType[] = [] + let refusal: Error | undefined + let acquireError: unknown const rig = await openAcpAdapterRig({ deps: { connect: (launch, options) => { @@ -151,7 +155,14 @@ describe("Orca's own refusal during a start", () => { child.stdin.once('finish', () => child.emit('exit', 0, null)) const connection = createAcpAgentConnection( launch, - { ...options, signIn: { read: readAntigravitySignInLine } }, + { + ...options, + onClose: (error) => { + refusal ??= error + options.onClose?.(error) + }, + signIn: { read: readAntigravitySignInLine } + }, () => child ) connections.push(connection) @@ -168,9 +179,21 @@ describe("Orca's own refusal during a start", () => { } }) try { - await restartWith((input) => rig.adapter.acquire(input)) + await restartWith((input) => + rig.adapter.acquire(input).catch((error: unknown) => { + acquireError = error + throw error + }) + ) const id = await accept('an agent whose sign-in line Orca cannot read') await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + // The adapter's own final verdict carries Orca's original refusal, not a generic close. + expect(refusal?.message).toContain('sign-in link Orca cannot read') + const verdict = findInErrorChain(acquireError, (cause) => + cause instanceof AgentSessionStartFailure ? cause : undefined + ) + expect(verdict?.retry).toBe('never') + expect(verdict?.cause).toBe(refusal) expect(connections).toHaveLength(1) expect(acquire).toHaveBeenCalledTimes(1) expect(dispatch).not.toHaveBeenCalled() From 334fd196dbb23887a3321274f4d9a0ad9c13cbc6 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:01:38 -0800 Subject: [PATCH 17/21] test(agent-session): give the worker retry fake the Codex adapter's typed start verdict The host now reads only the adapter's verdict on a failed start, so the fake open rejects with the verdict the Codex adapter returns for its declared overload instead of the raw request error. Co-Authored-By: Claude Opus 5.5 (1M context) --- ...hestration-structured-worker-retry.test.ts | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts index 2e53f107f97e..a2fd787011e5 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts @@ -7,6 +7,7 @@ import { import { HOST_TEST_NOW } from '../../../native-chat/agent-session-wire/structured-agent-session-host-test-data' import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import { CodexAppServerRequestError } from '../../../codex/app-server/codex-app-server-request-error' +import { codexStartFailure } from '../../../codex/app-server/codex-startup-outcome' import { OrchestrationDb } from '../../orchestration/db' import { createStructuredWorkerSession, @@ -48,6 +49,17 @@ afterEach(async () => { vi.useRealTimers() }) +// The Codex adapter's open answers with its typed verdict, which the host reads as given. +const overloaded = () => + codexStartFailure( + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) + ) + async function createWorker() { const workerRuntime = { ...runtimeStub(), @@ -112,14 +124,7 @@ it('starts only for its agent-sourced preamble and observes it after a transient const sessionId = created.identity.sessionId expect(rig.adapter.acquire).not.toHaveBeenCalled() expect((await rig.host.journalSnapshot(sessionId)).submissions).toEqual([]) - vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValueOnce( - new CodexAppServerRequestError( - 'initialize', - -32001, - 'Server overloaded; retry later.', - 'Server overloaded; retry later.' - ) - ) + vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValueOnce(overloaded()) const result = await deliverPreamble() expect(result.structuredTurnStart).toEqual({ verdict: 'observed' }) expect(rig.adapter.acquire).toHaveBeenCalledTimes(2) @@ -149,14 +154,7 @@ it('reports a preamble as undelivered after startup exhaustion and retires the f const { runtime, created, workerRuntime, deliverPreamble } = await createWorker() const sessionId = created.identity.sessionId expect(rig.store.listVisibleSessionIds()).toContain(sessionId) - vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValue( - new CodexAppServerRequestError( - 'initialize', - -32001, - 'Server overloaded; retry later.', - 'Server overloaded; retry later.' - ) - ) + vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValue(overloaded()) await expect(deliverPreamble()).rejects.toMatchObject({ code: 'dispatch_preamble_undelivered' }) expect(rig.adapter.acquire).toHaveBeenCalledTimes(5) expect(rig.adapter.dispatch).not.toHaveBeenCalled() From 9f43dd0066cdb1e4be3d47a521cb112629350e47 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:24:06 -0800 Subject: [PATCH 18/21] fix(agent-session): let Claude's own exit record decide a start the SDK reports as exited A Claude CLI missing from the launch PATH makes the supervisor exit 127 with the spawn error on stderr. The SDK rejects initialize with only "process exited with code 127", and the start took that report as its cause, so the missing CLI was lost: the chat said Claude stopped before it finished starting and the start was retried as an unanswered exit. The SDK's exit report is no answer; the start now reads Orca's own record of that exit, which carries the missing CLI. Also brings two ACP tests in line with the typed start verdict (a signed-out open rejects with the adapter's notSignedIn verdict) and the bounded drain after the agent stops reading. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../acp-structured-publish-at-spawn.test.ts | 6 ++- ...tructured-session-adapter-teardown.test.ts | 6 ++- .../claude-structured-session-startup-exit.ts | 21 +++++++- ...structured-session-startup-failure.test.ts | 54 ++++++++++++++++++- .../claude-structured-session-startup.ts | 10 ++-- 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/src/main/acp/acp-structured-publish-at-spawn.test.ts b/src/main/acp/acp-structured-publish-at-spawn.test.ts index da74d6cf7855..0d1d7b25457c 100644 --- a/src/main/acp/acp-structured-publish-at-spawn.test.ts +++ b/src/main/acp/acp-structured-publish-at-spawn.test.ts @@ -195,7 +195,11 @@ describe('ACP child published before its handshake', () => { acquisitionGeneration: acquired.acquisitionGeneration, signal: new AbortController().signal }) - ).rejects.toMatchObject({ reason: 'notSignedIn' }) + ).rejects.toMatchObject({ + retry: 'never', + fact: { kind: 'notSignedIn' }, + cause: { reason: 'notSignedIn' } + }) expect(rig.ended).toEqual([]) await rig.adapter.closeSession(SESSION) expect(rig.child().exited).toBe(true) diff --git a/src/main/acp/acp-structured-session-adapter-teardown.test.ts b/src/main/acp/acp-structured-session-adapter-teardown.test.ts index a453e5c27363..f3f215477e83 100644 --- a/src/main/acp/acp-structured-session-adapter-teardown.test.ts +++ b/src/main/acp/acp-structured-session-adapter-teardown.test.ts @@ -13,6 +13,7 @@ import { } from './acp-structured-adapter.test-support' import { framesOf, openAttachedHostRig, send } from './acp-structured-host.test-support' import { ACP_STOP_GRACE_MS } from './acp-structured-session-adapter-deps' +import { PROVIDER_EXIT_OUTPUT_DRAIN_MS } from '../provider-process/provider-exit-output-drain' afterEach(async () => { vi.useRealTimers() @@ -147,7 +148,10 @@ describe('ACP teardown with nothing to cancel', () => { return proves } child.agent.stdin.destroy() - await vi.waitFor(() => expect(child.closes).toBe(1)) + // The running prompt may still be answered, so the broken connection closes after the drain. + await vi.waitFor(() => expect(child.closes).toBe(1), { + timeout: PROVIDER_EXIT_OUTPUT_DRAIN_MS + 1_000 + }) proves = true await expect(rig.adapter.closeSession(SESSION)).resolves.toBe(true) expect(cancel).not.toHaveBeenCalled() diff --git a/src/main/claude/structured/claude-structured-session-startup-exit.ts b/src/main/claude/structured/claude-structured-session-startup-exit.ts index 2d2f187c5e96..3177847254c3 100644 --- a/src/main/claude/structured/claude-structured-session-startup-exit.ts +++ b/src/main/claude/structured/claude-structured-session-startup-exit.ts @@ -7,6 +7,7 @@ import { } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import type { ClaudeStreamJsonConnection } from '../claude-stream-json-connection' import type { StructuredAgentSessionEventSink } from '../../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { claudeStartupAnswer } from '../claude-startup-failure' /** A physical exit waits briefly for initialize's answer; a delivered answer outranks the exit. */ export async function claudeStartupAnswerAfterExit( @@ -22,7 +23,25 @@ export async function claudeStartupAnswerAfterExit( 2_000, exit ) - return connection.startupFailure?.() ?? (isClaudeQueryClosed(outcome) ? exit : outcome) + return ( + connection.startupFailure?.() ?? + (isClaudeQueryClosed(outcome) || reportsClaudeExit(outcome) ? exit : outcome) + ) +} + +/** The SDK's report that Claude's process ended is no answer; Orca's own record of that exit + * carries what ended it, such as a missing CLI, so a start reads that record instead. */ +export async function claudeExitForReport( + connection: Pick, + error: unknown +): Promise { + return reportsClaudeExit(error) && connection.exited + ? withTimeout(connection.exited, 2_000, error) + : error +} + +function reportsClaudeExit(error: unknown): boolean { + return error instanceof Error && claudeStartupAnswer(error) === 'exited' } /** An exit before publication can follow initialize's answer, which only publication reads. */ diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index 9f06ba2f6832..83e3736b371a 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -1,6 +1,7 @@ import sdkExit from '../fixtures/startup-sdk-process-exit.json' import { afterEach, expect, it, vi } from 'vitest' import { findInErrorChain } from '../../provider-process/provider-error-chain' +import { withMissingProviderExecutable } from '../../provider-process/provider-executable-missing' import { claudeStartupAnswerObservation } from '../claude-startup-answer-observation' import { runClaudeControl } from '../claude-agent-sdk-control-requests' import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' @@ -68,7 +69,12 @@ it.each(['control_request_failed', 'process_exited_nonzero'])( ) await vi.advanceTimersByTimeAsync(2_000) const error = await result - expect(error.cause.cause).toMatchObject({ errorClass, exitCode: 1 }) + // The SDK's report of the exit is no answer, so Orca's own exit record decides the start. + if (errorClass === 'process_exited_nonzero') { + expect(error.cause).toBe(exit) + } else { + expect(error.cause.cause).toMatchObject({ errorClass, exitCode: 1 }) + } expect(retryableStructuredAgentSessionStart(error)).toBe( errorClass === 'process_exited_nonzero' ) @@ -154,6 +160,52 @@ it('treats a turn that settled without an Orca dispatch as past startup on exit' await adapter.closeAll() }) +it("reads a missing CLI from Orca's exit when the SDK reports the exit first", async () => { + const claude = fakeClaude({ holdInit: true }) + const adapter = adapterAtPublishFor(claude) + const acquired = await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'missing-cli' + }) + const connection = claude.connections[0] + const exited = Promise.withResolvers() + connection.exited = exited.promise + // A supervised spawn ENOENT: the supervisor exits 127 and the SDK reports only that exit. + connection.initializationResult = () => + runClaudeControl( + 'initialize', + () => + Promise.reject( + Object.assign(new Error('Claude Code process exited with code 127'), { + errorClass: 'process_exited_nonzero', + exitCode: 127 + }) + ), + null + ) + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + await tick() + exited.resolve( + withMissingProviderExecutable( + withObservedProviderExit( + new Error('claude stream-json exited (code 127): spawn claude ENOENT') + ) + ) + ) + const error = await result + expect(error).toMatchObject({ retry: 'never', fact: { kind: 'cliMissing' } }) + claude.dispose() + await adapter.closeAll() +}) + it('settles an unanswered Claude startup exit with the same bounded recovery eligibility', async () => { vi.useFakeTimers() const claude = fakeClaude({ holdInit: true }) diff --git a/src/main/claude/structured/claude-structured-session-startup.ts b/src/main/claude/structured/claude-structured-session-startup.ts index 9c707edda2a3..01234501465b 100644 --- a/src/main/claude/structured/claude-structured-session-startup.ts +++ b/src/main/claude/structured/claude-structured-session-startup.ts @@ -31,7 +31,10 @@ import { readClaudeSettingsEffort } from './launch/claude-structured-session-options' import { claudeExecutionObservations } from './claude-structured-execution-observations' -import { claudeStartupAnswerAfterExit } from './claude-structured-session-startup-exit' +import { + claudeExitForReport, + claudeStartupAnswerAfterExit +} from './claude-structured-session-startup-exit' import type { ClaudeAuthDiagnostic, ClaudeSession, @@ -78,7 +81,7 @@ export async function readClaudeStartupFacts(input: { return result }) const initialization = await Promise.race([ - control.catch((error: unknown) => { + control.catch(async (error: unknown) => { try { input.initProof.seen() } catch (proofError) { @@ -89,7 +92,8 @@ export async function readClaudeStartupFacts(input: { throw input.connection.startupFailure?.() ?? proofError } } - throw input.connection.startupFailure?.() ?? error + const ended = await claudeExitForReport(input.connection, error) + throw input.connection.startupFailure?.() ?? ended }), Promise.race([ input.initProof.promise.then(() => new Promise(() => {})), From f394bed1207c3d2335b60f1708ce2a70f54e8238 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:47:43 -0800 Subject: [PATCH 19/21] perf(agent-session): stop parsing Claude's startup output once its failure is captured, and pin start and drain work at N and 10N A failed root result captured mid-chunk still let the rest of that chunk be framed and parsed; nothing reads those records once the start has its answer. The observer now accepts no further line once it has captured the failure, and drops what that chunk left in its framing state. Two contract tests drive the production start owner and transports at N and 10N: - starts through the host, router and Codex adapter (ready, final refusal, safe replay, deadline expiry): each start's owner, abort handles, tracked attempt, flight, buffered exits, adapter receipts, root-exit subscription and deadline timer return to baseline, a waiting start holds a fixed set, and spawns, opens, router reads and retirements, timers and failure rows grow exactly linearly; - ACP, JSON-lines and Codex connections with a pending call, answered at exit or abandoned after the agent stops reading (with duplicate terminal signals): each call settles once, each output drain finishes once and leaves no listener or timer; - an ACP start refused at sign-in keeps its own first cause and leaves no receipt or child. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../claude-startup-answer-observation.ts | 6 +- ...structured-session-startup-failure.test.ts | 13 +- ...agent-session-start-lifecycle-perf.test.ts | 399 ++++++++++++++++++ .../provider-exit-drain-scaling.test.ts | 226 ++++++++++ 4 files changed, 641 insertions(+), 3 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts create mode 100644 src/main/provider-process/provider-exit-drain-scaling.test.ts diff --git a/src/main/claude/claude-startup-answer-observation.ts b/src/main/claude/claude-startup-answer-observation.ts index e021dcda3f22..ec460aa09eb6 100644 --- a/src/main/claude/claude-startup-answer-observation.ts +++ b/src/main/claude/claude-startup-answer-observation.ts @@ -34,8 +34,9 @@ export function claudeStartupAnswerObservation(next?: (chunk: Buffer | string) = } } // The shared line budget bounds retention; a rejected prefix cannot prove a root record. + // A capture ends parsing mid-chunk; the rest of that chunk only reaches the next handler. const framer = createIncrementalNdjsonFramer(observe, () => {}, { - acceptLine: mayBeFailedResult + acceptLine: (line) => !done && mayBeFailedResult(line) }) return { failure: () => failure, @@ -44,6 +45,9 @@ export function claudeStartupAnswerObservation(next?: (chunk: Buffer | string) = feed: (chunk: Buffer | string): void => { if (!done) { framer.feed(typeof chunk === 'string' ? chunk : decoder.write(chunk)) + if (done) { + framer.reset() + } } next?.(chunk) } diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index 83e3736b371a..9f8a269f3207 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -325,7 +325,11 @@ it('parses only failed root results at startup, bounded by candidates rather tha : assistant(`working ${index}`) ) } - observation.feed(failedResult('error_during_execution')) + // Later failed results coalesced into the capturing chunk must not be parsed. + observation.feed( + failedResult('error_during_execution') + + failedResult('error_in_capturing_chunk').repeat(records) + ) await flush() const parsed = parse.mock.calls.map(([text]) => String(text)) const failure = observation.failure() @@ -336,8 +340,12 @@ it('parses only failed root results at startup, bounded by candidates rather tha observation.feed(failedResult('error_after_finish')) await flush() return { - irrelevantParses: parsed.filter((text) => !text.includes('error_during_execution')).length, + irrelevantParses: parsed.filter( + (text) => !text.includes('error_during_execution') && !text.includes('capturing_chunk') + ).length, candidateParses: parsed.filter((text) => text.includes('error_during_execution')).length, + sameChunkParsesAfterCapture: parsed.filter((text) => text.includes('capturing_chunk')) + .length, failure, parsesAfterCapture: parse.mock.calls.length } @@ -350,6 +358,7 @@ it('parses only failed root results at startup, bounded by candidates rather tha expect(n).toMatchObject({ irrelevantParses: 0, candidateParses: 1, + sameChunkParsesAfterCapture: 0, failure: expect.objectContaining({ errorClass: 'error_result' }), parsesAfterCapture: 0 }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts new file mode 100644 index 000000000000..7fcae2044bb3 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts @@ -0,0 +1,399 @@ +import { getEventListeners } from 'node:events' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { createAcpAgentConnection } from '../../acp/acp-agent-connection' +import { openAcpAdapterRig } from '../../acp/acp-structured-adapter.test-support' +import { readAntigravitySignInLine } from '../../antigravity/antigravity-sign-in-url' +import { findInErrorChain } from '../../provider-process/provider-error-chain' +import { createFakePipedChild } from '../../../shared/__fixtures__/fake-spawned-child' +import { closeProviderTimelineRigs } from '../agent-session-timeline/provider-timeline-assembler-test-support' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' +import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { + adapterFor, + fakeCodex, + THREAD_ID +} from '../../codex/codex-structured-session-adapter-fixture' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session/records/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { + claudeAndCodexAgents, + claudeAndCodexRouter +} from './structured-agent-session-adapter-router-test-support' +import type * as ExitObservation from './structured-agent-session-exit-observation' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' +import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' +import { restTestSend } from './structured-agent-session-rest-test-rig' +import type * as StartFailure from './structured-agent-session-start-failure' + +// Every adapter's startup receipts and every acquisition's root-exit receipt, to count what they +// still hold after each start, whichever adapter or sink created them. +const created = vi.hoisted(() => ({ + endings: new Array(), + rootExits: new Array() +})) +const retainedReceipts = () => + created.endings.reduce((sum, endings) => sum + endings['endings'].size, 0) +vi.mock('./structured-agent-session-start-failure', async (original) => { + const actual = await original() + class CountedEndings extends actual.AgentSessionStartEndings { + constructor() { + super() + created.endings.push(this) + } + } + return { ...actual, AgentSessionStartEndings: CountedEndings } +}) +vi.mock('./structured-agent-session-exit-observation', async (original) => { + const actual = await original() + class CountedRootExit extends actual.StructuredAgentSessionRootExitReceipt { + constructor() { + super() + created.rootExits.push(this) + } + } + return { ...actual, StructuredAgentSessionRootExitReceipt: CountedRootExit } +}) + +const CALLER = { callerKey: 'client-1' } +const CEILING_MS = 20_000 +type Cycle = 'ready' | 'final' | 'retried' | 'expired' +const CYCLES: readonly Cycle[] = ['ready', 'final', 'retried', 'expired'] +const SCRIPTS: Record = { + ready: ['ok'], + final: ['refuse'], + retried: ['hang', 'ok'], + expired: ['hang'] +} + +let root: string +beforeEach(async () => { + vi.useFakeTimers({ + toFake: ['Date', 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval'] + }) + root = await mkdtemp(join(tmpdir(), 'orca-start-lifecycle-')) + resetHostTestOperationIds() + created.endings.length = 0 + created.rootExits.length = 0 +}) +afterEach(async () => { + try { + await closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) + } finally { + vi.useRealTimers() + vi.restoreAllMocks() + } +}) + +/** Runs `count` rounds of each start outcome through the production host, router and Codex adapter. */ +async function runStarts(count: number) { + // Each thread/start or thread/resume takes the next scripted answer. + const answers: ('ok' | 'refuse' | 'hang')[] = [] + const threadAnswer = () => { + const answer = answers.shift() + if (answer === 'refuse') { + throw new CodexAppServerRequestError('thread/start', -32600, 'invalid', 'invalid') + } + return answer === 'hang' + ? new Promise(() => {}) + : { thread: { id: THREAD_ID }, model: 'gpt-live', reasoningEffort: 'medium' } + } + const codex = fakeCodex({ 'thread/start': threadAnswer, 'thread/resume': threadAnswer }) + let host: StructuredAgentSessionHost | undefined + const store = await openTestAgentSessionRecordStore(join(root, `starts-${count}`)) + const adapter = adapterFor(codex, {}, [], { + // Once a thread exists, each new child resumes it. + resolveLaunch: async () => ({ + command: 'codex', + args: ['app-server'], + cwd: '/work/repo', + codexHome: null, + resumeThreadId: store.getRecord(SESSION)?.providerHandleChain.length ? THREAD_ID : null + }), + onEvent: (event) => { + if (event.type === 'ended' && 'cause' in event) { + void host?.handleAdapterEvent(event) + } + } + }) + const other = adapterFor(fakeCodex()) + const router = claudeAndCodexRouter({ claude: other, codex: adapter }, async () => { + await adapter.closeAll() + await other.closeAll() + }) + const dispatch = vi.fn(async () => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD_ID, + turnId: `turn-${dispatch.mock.calls.length}`, + ordinal: dispatch.mock.calls.length + } + })) + // Turn work is not under test; every start-path call reaches the production router. + Object.assign(router, { dispatch }) + host = new StructuredAgentSessionHost({ + agents: claudeAndCodexAgents({ claude: other, codex: adapter }), + logger: createStructuredAgentSessionLogger(), + store, + adapter: router, + journalDatabase: openTestJournalHostDatabase(join(root, `starts-${count}`)), + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${codex.connections.length}`, + startupLimits: { ceilingMs: CEILING_MS }, + idleSweep: { intervalMs: 60 * 60_000, idleMs: 60 * 60_000 }, + now: () => NOW + }) + const { runtimeState } = host.collaboratorsForTests() + const spies = { + opens: vi.spyOn(router, 'open'), + // The router reaches every registered adapter by exact generation. + codexEndingReads: vi.spyOn(adapter, 'startupEnding'), + otherEndingReads: vi.spyOn(other, 'startupEnding'), + codexRetirements: vi.spyOn(adapter, 'retireStartup'), + otherRetirements: vi.spyOn(other, 'retireStartup'), + timersScheduled: vi.spyOn(globalThis, 'setTimeout') + } + // Abort handles the host still holds, and the listeners on their signals. + const abortSignals = () => + [...runtimeState.acquireAborts['inFlight'].values()].flatMap((waits) => + [...waits].map(({ controller }) => controller.signal) + ) + const abortListeners = () => + abortSignals().reduce((sum, signal) => sum + getEventListeners(signal, 'abort').length, 0) + const held = () => ({ + timers: vi.getTimerCount(), + abortListeners: abortListeners(), + abortHandles: abortSignals().length, + trackedStarts: runtimeState.startupAttempts['open'].size, + owners: runtimeState.startupAttempts['owners'].size, + flights: runtimeState.startFlights['flying'].size, + bufferedExits: runtimeState.startFlights['exits'].size, + receipts: retainedReceipts(), + rootExitListeners: created.rootExits.reduce((sum, exit) => sum + exit['listeners'].size, 0), + liveChildren: codex.connections.filter((connection) => !connection.closed).length + }) + const eventually = (assertion: () => unknown) => + vi.waitFor(assertion, { timeout: 10_000, interval: 0 }) + + const submission = async (id: string) => + (await host!.journalSnapshot(SESSION)).submissions.find((row) => row.clientMessageId === id) + const failureRows = async () => + (await host!.journalSnapshot(SESSION)).items.filter( + (item) => item.body.kind === 'status' && item.body.tone === 'error' + ).length + const phase = () => host!.collaboratorsForTests().sessions.get(SESSION)?.child?.phase + const exitChild = (connection = codex.connections.at(-1)!) => { + connection.closed = true + const exit = new CodexAppServerExitError('codex app-server connection ended') + // A duplicate terminal report must change nothing. + connection.handlers.onExit?.(exit, { expected: false }) + connection.handlers.onExit?.(exit, { expected: false }) + } + + // Work the starts did, counted from a ready chat. + const work = async (): Promise> => ({ + ...Object.fromEntries( + Object.entries(spies).map(([field, spy]) => [field, spy.mock.calls.length]) + ), + spawns: codex.connections.length, + failureRows: await failureRows() + }) + + answers.push('ok') + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + await eventually(() => expect(phase()).toBe('ready')) + const begun = await work() + const after: Record[]> = { + ready: [], + final: [], + retried: [], + expired: [] + } + let rows = 0 + // What one start holds while its child is still answering. + const starting: ReturnType[] = [] + for (let round = 0; round < count; round++) { + for (const cycle of CYCLES) { + // The chat's ready child dies, so the next message starts a new one as the script says. + if (phase() === 'ready') { + exitChild() + await eventually(() => expect(phase()).toBeUndefined()) + } + await host.flushStreamedEvents(SESSION) + rows = await failureRows() + const spawned = codex.connections.length + answers.push(...SCRIPTS[cycle]) + const send = restTestSend(cycle, store.getRecord(SESSION)!.lease.runtimeFence) + expect(await host.send(CALLER, send), `${cycle} ${round}`).toMatchObject({ ok: true }) + if (cycle === 'retried') { + await eventually(() => expect(answers).toEqual(['ok'])) + starting.push(held()) + exitChild() + } + if (cycle === 'expired') { + await eventually(() => expect(answers).toEqual([])) + await vi.advanceTimersByTimeAsync(CEILING_MS) + } + // The start's owner decides it: the message runs on a ready child, or one row says why not. + const started = cycle === 'ready' || cycle === 'retried' + await eventually(async () => { + await vi.advanceTimersByTimeAsync(100) + expect(runtimeState.startFlights.has(SESSION)).toBe(false) + expect(phase(), `${cycle} ${round}`).toBe(started ? 'ready' : undefined) + expect((await submission(send.envelope.clientOperationId))?.dispatchState).toBe( + started ? 'accepted' : 'rejected' + ) + expect(await failureRows()).toBe(rows + (started ? 0 : 1)) + }) + expect(codex.connections).toHaveLength(spawned + (cycle === 'retried' ? 2 : 1)) + await host.flushStreamedEvents(SESSION) + await vi.advanceTimersByTimeAsync(2_000) + after[cycle].push(held()) + } + } + const done = await work() + await host.close(SESSION, 'evict') + await host.flushAllStreamedEvents() + await router.closeAll() + vi.restoreAllMocks() + const counts = Object.fromEntries( + Object.entries(done).map(([field, count]) => [field, count - begun[field]]) + ) + return { after, starting, counts } +} + +it('returns every start resource to baseline and does constant work per start, at N and 10N', async () => { + const n = await runStarts(1) + const tenN = await runStarts(10) + for (const cycle of CYCLES) { + const [first] = n.after[cycle] + // Nothing a finished start owned survives it: no owner, abort handle, tracked attempt, flight, + // buffered exit, adapter receipt or root-exit listener, and no exited child left running. + expect(first, cycle).toMatchObject({ + abortHandles: 0, + trackedStarts: 0, + owners: 0, + flights: 0, + bufferedExits: 0, + receipts: 0, + rootExitListeners: 0, + liveChildren: cycle === 'ready' || cycle === 'retried' ? 1 : 0 + }) + // Timers and abort listeners held after each start do not grow with the starts before it. + for (const held of tenN.after[cycle]) { + expect(held, cycle).toEqual(first) + } + } + // While a start waits on its child it holds a fixed set, however many starts came before it: + // one deadline timer, the flight's and the owner's abort handles, one tracked attempt, owner, + // receipt and root-exit subscription. + expect(n.starting[0]).toMatchObject({ + timers: n.after.ready[0].timers + 1, + abortHandles: 2, + trackedStarts: 1, + owners: 1, + flights: 1, + receipts: 1, + rootExitListeners: 1 + }) + for (const held of tenN.starting) { + expect(held).toEqual(n.starting[0]) + } + // Per round of four starts: five spawns and opens (the retried start opens twice) and a failure + // row for the final and the expired start. Router reads and retirements reach the registered + // adapters a fixed number of times per open, whatever the starts before it. + expect(n.counts).toMatchObject({ + spawns: 5, + opens: 5, + codexEndingReads: 4, + otherEndingReads: 4, + codexRetirements: 5, + otherRetirements: 5, + failureRows: 2 + }) + for (const [field, count] of Object.entries(n.counts)) { + expect(tenN.counts[field], field).toBe(count * 10) + } +}) + +/** Starts an ACP agent that prints a sign-in line Orca cannot read, `count` times. */ +async function refuseAcpStarts(count: number) { + const connections: ReturnType[] = [] + const refusals = new Map, Error>() + const rig = await openAcpAdapterRig({ + deps: { + connect: (launch, options) => { + const child = Object.assign(createFakePipedChild(), { + pid: 9_999_999, + stdout: new PassThrough(), + stdin: new PassThrough(), + stderr: new PassThrough(), + kill: () => true + }) + // Orca's cleanup ends stdin; the agent then exits. + child.stdin.once('finish', () => child.emit('exit', 0, null)) + const connection: ReturnType = createAcpAgentConnection( + launch, + { + ...options, + onClose: (error) => { + if (!refusals.has(connection)) { + refusals.set(connection, error) + } + options.onClose?.(error) + }, + signIn: { read: readAntigravitySignInLine } + }, + () => child + ) + connections.push(connection) + child.stdout.write('Open the following link to authenticate the ACP server: not-a-url\n') + return connection + } + } + }) + const retained: number[] = [] + try { + for (let index = 0; index < count; index++) { + const error = await rig.acquire({ waitForStart: false }).catch((caught: unknown) => caught) + const connection = connections[index] + // The verdict is final and carries this connection's own first refusal, not a later close. + const verdict = findInErrorChain(error, (cause) => + cause instanceof AgentSessionStartFailure ? cause : undefined + ) + expect(verdict?.retry).toBe('never') + expect(verdict?.cause).toBe(refusals.get(connection)) + expect(connection.lostBeforeExit).toBe(refusals.get(connection)) + expect(connection.exited).toBe(true) + retained.push(retainedReceipts()) + } + return { spawns: connections.length, retained } + } finally { + await rig.adapter.closeAll() + await closeProviderTimelineRigs() + } +} + +it('ACP: each refused start keeps its own cause and leaves no receipt or child, at N and 10N', async () => { + vi.useRealTimers() + for (const count of [10, 100]) { + const { spawns, retained } = await refuseAcpStarts(count) + expect(spawns).toBe(count) + expect(retained).toEqual(Array(count).fill(0)) + } +}) diff --git a/src/main/provider-process/provider-exit-drain-scaling.test.ts b/src/main/provider-process/provider-exit-drain-scaling.test.ts new file mode 100644 index 000000000000..afc92cfd274b --- /dev/null +++ b/src/main/provider-process/provider-exit-drain-scaling.test.ts @@ -0,0 +1,226 @@ +import type { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' +import { afterEach, expect, it, vi } from 'vitest' +import type { spawnProcess } from '@orca/process-host' +import type { PipedProcessSpawner } from '@orca/process-host/process-spec' +import { createFakePipedChild } from '../../shared/__fixtures__/fake-spawned-child' +import { createAcpAgentConnection } from '../acp/acp-agent-connection' +import { AcpScriptedAgent, type FakeFrame } from '../acp/acp-scripted-agent.test-support' +import { openCodexAppServerConnection } from '../codex/app-server/codex-app-server-connection' +import { JsonlRpcAgentConnection } from '../jsonl-rpc/agent-connection' +import { PROVIDER_EXIT_OUTPUT_DRAIN_MS } from './provider-exit-output-drain' +import type * as OutputDrain from './provider-exit-output-drain' +import type { terminateProviderProcessTree } from './provider-process-teardown' + +// Every output drain any transport starts, the listeners it attached, and whether it finished. +type Attached = { emitter: EventEmitter; listener: ReturnType[number] } +const drains = vi.hoisted(() => new Array<{ attached: Attached[]; finished: number }>()) +vi.mock('./provider-exit-output-drain', async (original) => { + const actual = await original() + const watched = (stdout: EventEmitter, child?: EventEmitter): Attached[] => + [ + { emitter: stdout, event: 'end' }, + { emitter: stdout, event: 'close' }, + ...(child ? [{ emitter: child, event: 'close' }] : []) + ].flatMap(({ emitter, event }) => + emitter.listeners(event).map((listener) => ({ emitter, listener })) + ) + return { + ...actual, + afterProviderOutputDrains: ( + ...[stdout, finish, child]: Parameters + ) => { + const before = new Set(watched(stdout, child).map(({ listener }) => listener)) + const drain: { attached: Attached[]; finished: number } = { attached: [], finished: 0 } + drains.push(drain) + const done = actual.afterProviderOutputDrains( + stdout, + () => { + drain.finished += 1 + finish() + }, + child + ) + drain.attached = watched(stdout, child).filter(({ listener }) => !before.has(listener)) + return done + } + } +}) +vi.mock('./provider-process-teardown', () => ({ + terminateProviderProcessTree: vi.fn( + async () => 'unverifiable' + ) +})) + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +type Child = ReturnType +/** One connection over a fake child, with a call the agent has not answered yet. */ +type Opened = { + child: Child + pending: Promise + /** The agent's own answer to that call, written to its stdout. */ + answer: () => void + close: () => Promise + lostBeforeExit?: () => Error | undefined +} + +function fakeChild(stdout = new PassThrough(), stdin = new PassThrough()) { + return Object.assign(createFakePipedChild(), { + // Outside any real process table so teardown never mistakes an unrelated process for it. + pid: 9_999_999, + stdin, + stdout, + stderr: new PassThrough(), + kill: () => true + }) +} + +const TRANSPORTS: Record Promise> = { + acp: async () => { + const agent = new AcpScriptedAgent() + const child = fakeChild(agent.stdout, agent.stdin) + let prompt: FakeFrame | undefined + agent.on('initialize', (frame) => agent.reply(frame, { protocolVersion: 1 })) + agent.on('session/new', (frame) => agent.reply(frame, { sessionId: 'session-1' })) + agent.on('session/prompt', (frame) => { + prompt = frame + }) + const spawn: PipedProcessSpawner = () => child + const connection = createAcpAgentConnection( + { command: 'fixture-acp-agent', args: ['--acp'], cwd: '/execution-host/folder', env: {} }, + {}, + spawn + ) + await connection.start({ cwd: '/execution-host/folder', mcpServers: [] }) + const pending = connection.prompt([{ type: 'text', text: 'hello' }]) + await vi.waitFor(() => expect(prompt).toBeDefined()) + return { + child, + pending, + answer: () => agent.reply(prompt!, { stopReason: 'end_turn' }), + close: () => connection.close(), + lostBeforeExit: () => connection.lostBeforeExit + } + }, + jsonl: async () => { + const child = fakeChild() + const spawn: PipedProcessSpawner = () => child + const connection = new JsonlRpcAgentConnection( + { command: 'fixture-rpc-agent', args: ['--mode', 'rpc'], cwd: '/folder', env: {} }, + {}, + spawn + ) + return { + child, + pending: connection.request('get_state'), + answer: () => + child.stdout.write( + '{"type":"response","command":"get_state","id":"orca-1","success":true,"data":{}}\n' + ), + close: () => connection.close() + } + }, + codex: async () => { + const child = fakeChild() + child.stdin.once('data', () => child.stdout.write(`${JSON.stringify({ id: 1, result: {} })}\n`)) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The connection reads only events, pid, streams and kill from this stub. + const spawnImpl = (() => child) as unknown as typeof spawnProcess + const connection = await openCodexAppServerConnection( + { command: 'codex', args: ['app-server'] }, + {}, + spawnImpl + ) + return { + child, + pending: connection.request('thread/start', {}), + answer: () => child.stdout.write(`${JSON.stringify({ id: 2, result: {} })}\n`), + close: () => connection.close() + } + } +} + +/** Listeners an output drain attached that are still attached. */ +const drainListeners = () => + drains.reduce( + (sum, drain) => + sum + + drain.attached.filter(({ emitter, listener }) => + [...emitter.listeners('end'), ...emitter.listeners('close')].includes(listener) + ).length, + 0 + ) + +/** Runs `count` connections through each terminal order and counts what each one leaves. */ +async function drainCycles(open: () => Promise, count: number) { + drains.length = 0 + const timers = vi.spyOn(globalThis, 'setTimeout') + const settled = { answered: 0, failed: 0 } + const left: { timers: number; drainListeners: number }[] = [] + for (let cycle = 0; cycle < count; cycle++) { + for (const answered of [true, false]) { + const opened = await open() + const { child } = opened + opened.pending.then( + () => (settled.answered += 1), + () => (settled.failed += 1) + ) + if (answered) { + // The exit is seen before the answer written just before it; the drain waits for it. + child.emit('exit', 0, null) + child.emit('exit', 0, null) + opened.answer() + child.stdout.end() + } else { + // The agent stops reading, twice over, and exits with its stdout still held open. + const lost = new Error('broken pipe') + child.stdin.emit('error', lost) + child.stdin.emit('error', new Error('a later failure')) + child.stdin.emit('close') + child.emit('exit', 1, null) + child.emit('exit', 1, null) + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS) + // ACP keeps why the agent stopped reading, its first reason only, for its exit's readers. + expect(opened.lostBeforeExit?.() ?? lost).toBe(lost) + } + await vi.advanceTimersByTimeAsync(0) + // Read before the child's own close, which would fire any listener a drain left behind. + const listening = drainListeners() + await opened.close() + child.emit('close', 0, null) + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS) + left.push({ timers: vi.getTimerCount(), drainListeners: listening }) + } + } + const timersScheduled = timers.mock.calls.length + timers.mockRestore() + const finished = drains.map((drain) => drain.finished) + return { settled, left, timersScheduled, drains: drains.length, finished } +} + +it.each(Object.keys(TRANSPORTS))( + '%s: settles each pending call once and leaves no drain timer or listener, at N and 10N', + async (transport) => { + vi.useFakeTimers() + const n = await drainCycles(TRANSPORTS[transport], 10) + const tenN = await drainCycles(TRANSPORTS[transport], 100) + for (const run of [n, tenN]) { + for (const held of run.left) { + expect(held).toEqual({ timers: 0, drainListeners: 0 }) + } + } + // An answer written before exit wins; an unanswered call fails once, at the drain bound. + expect(n.settled).toEqual({ answered: 10, failed: 10 }) + expect(tenN.settled).toEqual({ answered: 100, failed: 100 }) + expect(tenN.timersScheduled).toBe(10 * n.timersScheduled) + // Each drain finishes exactly once, and a connection starts a fixed number of them. + for (const run of [n, tenN]) { + expect(run.finished.every((finished) => finished === 1)).toBe(true) + } + expect(n.drains).toBeGreaterThan(0) + expect(tenN.drains).toBe(10 * n.drains) + } +) From 2be8d5bf4a16245f73356c00e7b2f8887118a67c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:02:55 -0800 Subject: [PATCH 20/21] fix(agent-session): decide a failed start once from everything its adapter saw A failed start was decided from whichever error reached the decision first, so a later or poorer report could replace the one that explained it: the SDK's exit report hid the missing CLI Orca's own exit record carried, and an output drain let a load deadline give way to an unanswered exit. One shared rule now weighs every observation an adapter kept. The start may run again only if every observation allows it, and the most specific one explains it: what the user can act on, then a request deadline, then the agent's answer, then Orca's record of its exit, then an SDK's report of that exit. Claude, Codex and Pi pass their exit records alongside the operation's error instead of choosing between them. An ACP agent that exited while its output drained no longer gets a replacement session after its load fails. JSON-lines output that reports both its end and its close now starts one wait, so the agent's exit can always cancel it. Contracts at N and 10N: JSON-lines output ending while its root lives, real Claude and Pi adapters retiring every start's receipt, and a user Stop whose cleanup is held then released. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../acp/acp-session-reopen-failure.test.ts | 94 +++++--- src/main/acp/acp-structured-handshake.ts | 3 +- src/main/claude/claude-startup-failure.ts | 4 +- .../claude-structured-session-startup-exit.ts | 43 ++-- ...structured-session-startup-failure.test.ts | 88 ++++--- .../claude-structured-session-startup.ts | 19 +- .../codex/codex-structured-start-verdict.ts | 3 +- src/main/jsonl-rpc/agent-connection.ts | 4 + .../structured-agent-session-start-failure.ts | 85 +++++-- ...agent-session-start-lifecycle-perf.test.ts | 223 ++++++++++++++++-- ...agent-session-start-retry-recorded.test.ts | 197 +++++++++++++++- src/main/pi/rpc-startup-open.ts | 18 +- .../provider-exit-drain-scaling.test.ts | 67 ++++++ 13 files changed, 691 insertions(+), 157 deletions(-) diff --git a/src/main/acp/acp-session-reopen-failure.test.ts b/src/main/acp/acp-session-reopen-failure.test.ts index 21da77a1466a..25acf5cae460 100644 --- a/src/main/acp/acp-session-reopen-failure.test.ts +++ b/src/main/acp/acp-session-reopen-failure.test.ts @@ -1,9 +1,12 @@ import { openAcpAdapterRig } from './acp-structured-adapter.test-support' import { closeProviderTimelineRigs } from '../native-chat/agent-session-timeline/provider-timeline-assembler-test-support' import { afterEach, expect, it, vi } from 'vitest' +import { PassThrough } from 'node:stream' +import { createFakePipedChild } from '../../shared/__fixtures__/fake-spawned-child' +import { PROVIDER_EXIT_OUTPUT_DRAIN_MS } from '../provider-process/provider-exit-output-drain' +import { createAcpAgentConnection } from './acp-agent-connection' import { AcpAgentError, AcpRequestTimeoutError } from './acp-errors' -import { openAcpStart } from './acp-structured-start-open' -import { AgentSessionStartFailure } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { AcpScriptedAgent } from './acp-scripted-agent.test-support' import { agentSessionFailureFact } from '../../shared/agent-session/failures/agent-session-failure' import { agentSessionFailureWords } from '../../shared/agent-session/failures/agent-session-failure-words' import { acpReopenTakeover } from './acp-session-reopen-failure' @@ -29,7 +32,10 @@ it.each(['ServerUnavailable', 'CatalogNotReady'])( } ) -afterEach(closeProviderTimelineRigs) +afterEach(async () => { + vi.useRealTimers() + await closeProviderTimelineRigs() +}) it('propagates a temporary load envelope before takeover or a dead-child assertion can replace it', async () => { const rig = await openAcpAdapterRig({ @@ -57,37 +63,59 @@ it('propagates a temporary load envelope before takeover or a dead-child asserti }) it("keeps a load deadline's words when the agent exited while its output drained", async () => { - const timeout = new AcpRequestTimeoutError('session/load', 300_000) - // The agent exited unanswered at 299.5 s; the request's own deadline fired while its output drained. - const child = { - fence: 3, - acquisitionGeneration: 'g1', - ended: false, - closeRequested: false, - connection: { exited: true }, - spec: { agent: 'opencode', dialect: 'opencode' }, - open: async () => { - throw timeout + vi.useFakeTimers() + const spawned: ReturnType[] = [] + const agents: AcpScriptedAgent[] = [] + const rig = await openAcpAdapterRig({ + launch: { + resume: { + key: 'saved-key', + sessionId: 'saved-id', + mayBeUnsaved: () => false, + unannouncedLosses: () => [] + } + }, + deps: { + connect: (launch, options) => { + const agent = new AcpScriptedAgent() + agent.on('initialize', (frame) => + agent.reply(frame, { protocolVersion: 1, agentCapabilities: { loadSession: true } }) + ) + // The agent never answers the load. + agent.on('session/load', () => {}) + const child = Object.assign(createFakePipedChild(), { + pid: 9_999_999, + stdout: agent.stdout, + stdin: agent.stdin, + stderr: new PassThrough(), + kill: () => true + }) + spawned.push(child) + agents.push(agent) + return createAcpAgentConnection(launch, options, () => child) + } } - } - const failure = await openAcpStart( - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: openAcpStart reads only the starting child's fields set above. - { sessions: new Map([['s1', child]]) } as unknown as Parameters[0], - { sessionId: 's1', fence: 3, acquisitionGeneration: 'g1', signal: new AbortController().signal } - ).catch((error: unknown) => error) - expect(failure).toMatchObject({ - retry: 'never', - fact: { kind: 'startFailed', requestTimeout: { method: 'session/load', timeoutMs: 300_000 } } }) - if (!(failure instanceof AgentSessionStartFailure)) { - throw new Error('the open did not fail with its verdict') - } - expect(agentSessionFailureWords(failure.fact, { surface: 'rejection' })).toEqual( - agentSessionFailureWords( - agentSessionFailureFact('startFailed', { - requestTimeout: { method: 'session/load', timeoutMs: 300_000 } - }), - { surface: 'rejection' } + try { + const acquired = rig.acquire() + await vi.advanceTimersByTimeAsync(299_500) + // The agent exits unanswered at 299.5 s, holding stdout open; the load's own deadline fires at + // 300 s, inside the output drain that the exit started. + spawned[0].emit('exit', 1, null) + await vi.advanceTimersByTimeAsync(500 + PROVIDER_EXIT_OUTPUT_DRAIN_MS) + await acquired + const timedOut = agentSessionFailureFact('startFailed', { + requestTimeout: { method: 'session/load', timeoutMs: 300_000 } + }) + expect(rig.openVerdict()).toMatchObject({ retry: 'never', fact: timedOut }) + expect(rig.openFailure()).toBeInstanceOf(AcpRequestTimeoutError) + expect(agentSessionFailureWords(rig.openVerdict()!.fact, { surface: 'rejection' })).toEqual( + agentSessionFailureWords(timedOut, { surface: 'rejection' }) ) - ) + // No replacement session is opened in an agent that already exited. + expect(agents[0].frames.map((frame) => frame.method)).not.toContain('session/new') + expect(spawned).toHaveLength(1) + } finally { + await rig.adapter.closeAll() + } }) diff --git a/src/main/acp/acp-structured-handshake.ts b/src/main/acp/acp-structured-handshake.ts index c743a05c67ac..3ba1b4f1dcfb 100644 --- a/src/main/acp/acp-structured-handshake.ts +++ b/src/main/acp/acp-structured-handshake.ts @@ -88,7 +88,8 @@ export async function initializeAcpStructuredSession( attaching.translator.finishLoad() } catch (error) { takeover = acpReopenTakeover(error, resume, { - over: connection.closed || input.abandoned(), + // An agent already exited, its output still draining, has no session to take over in. + over: connection.closed || connection.exited || input.abandoned(), now: now(), warn: (fields) => deps.logger?.warn(ACP_REOPEN_FAILED, { ...fields, sessionId }) }) diff --git a/src/main/claude/claude-startup-failure.ts b/src/main/claude/claude-startup-failure.ts index 0727cc9f46bd..f7eeec57316b 100644 --- a/src/main/claude/claude-startup-failure.ts +++ b/src/main/claude/claude-startup-failure.ts @@ -15,7 +15,7 @@ export function claudeStartupErrorClass(error: Error): string | undefined { : undefined } -/** The SDK's tag says whether Claude's process ended unanswered; every other tag is its answer. +/** The SDK's tag reports whether Claude's process ended unanswered; every other tag is its answer. * A launch the host could not run for want of resources is left to the shared rule. */ export const claudeStartupAnswer: AgentSessionStartAnswer = (error) => { const tag = @@ -27,7 +27,7 @@ export const claudeStartupAnswer: AgentSessionStartAnswer = (error) => { return undefined } if (tag === 'process_exited_nonzero' || tag === 'process_killed_by_signal') { - return 'exited' + return 'reported' } if (tag === 'executable_launch_failed' || tag === 'spawn_failed' || tag === 'process_error') { return providerResourcePressure(error) ? undefined : 'final' diff --git a/src/main/claude/structured/claude-structured-session-startup-exit.ts b/src/main/claude/structured/claude-structured-session-startup-exit.ts index 3177847254c3..76007297049a 100644 --- a/src/main/claude/structured/claude-structured-session-startup-exit.ts +++ b/src/main/claude/structured/claude-structured-session-startup-exit.ts @@ -7,41 +7,40 @@ import { } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import type { ClaudeStreamJsonConnection } from '../claude-stream-json-connection' import type { StructuredAgentSessionEventSink } from '../../native-chat/agent-session-wire/structured-agent-session-event-sink' -import { claudeStartupAnswer } from '../claude-startup-failure' +import { claudeStartFailure, claudeStartupAnswer } from '../claude-startup-failure' +import type { AgentSessionStartFailure } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' -/** A physical exit waits briefly for initialize's answer; a delivered answer outranks the exit. */ +/** A physical exit waits briefly for initialize's answer, which the start's verdict weighs + * against the exit; a query closed by that exit is no answer. */ export async function claudeStartupAnswerAfterExit( connection: Pick, control: Promise, exit: unknown -): Promise { +): Promise { const outcome = await withTimeout( control.then( - () => exit, + () => undefined, (error: unknown) => error ), 2_000, - exit - ) - return ( - connection.startupFailure?.() ?? - (isClaudeQueryClosed(outcome) || reportsClaudeExit(outcome) ? exit : outcome) + undefined ) + return claudeStartFailure([ + connection.startupFailure?.(), + isClaudeQueryClosed(outcome) ? undefined : outcome, + exit + ]) } -/** The SDK's report that Claude's process ended is no answer; Orca's own record of that exit - * carries what ended it, such as a missing CLI, so a start reads that record instead. */ +/** Orca's own record of the exit an SDK report describes, which carries what ended it, such as a + * missing CLI; the start's verdict weighs it against the report. */ export async function claudeExitForReport( connection: Pick, error: unknown ): Promise { - return reportsClaudeExit(error) && connection.exited - ? withTimeout(connection.exited, 2_000, error) - : error -} - -function reportsClaudeExit(error: unknown): boolean { - return error instanceof Error && claudeStartupAnswer(error) === 'exited' + return error instanceof Error && claudeStartupAnswer(error) === 'reported' && connection.exited + ? withTimeout(connection.exited, 2_000, undefined) + : undefined } /** An exit before publication can follow initialize's answer, which only publication reads. */ @@ -53,9 +52,11 @@ export async function claudeStartupError( if (refusal || !connection) { return refusal ?? error } - return providerExitObserved(error) - ? claudeStartupAnswerAfterExit(connection, connection.initializationResult(), error) - : (connection.startupFailure?.() ?? error) + if (providerExitObserved(error)) { + return claudeStartupAnswerAfterExit(connection, connection.initializationResult(), error) + } + const result = connection.startupFailure?.() + return result ? claudeStartFailure([result, error]) : error } /** An unexpected exit, and when it ends the start: a delivered initialize answer outranks it. The diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts index 9f8a269f3207..e80f775ee6dc 100644 --- a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -160,51 +160,61 @@ it('treats a turn that settled without an Orca dispatch as past startup on exit' await adapter.closeAll() }) -it("reads a missing CLI from Orca's exit when the SDK reports the exit first", async () => { - const claude = fakeClaude({ holdInit: true }) - const adapter = adapterAtPublishFor(claude) - const acquired = await adapter.acquire({ - identity: identityFor(), - fence: 7, - spawnToken: 'missing-cli' - }) - const connection = claude.connections[0] - const exited = Promise.withResolvers() - connection.exited = exited.promise - // A supervised spawn ENOENT: the supervisor exits 127 and the SDK reports only that exit. - connection.initializationResult = () => - runClaudeControl( - 'initialize', - () => - Promise.reject( - Object.assign(new Error('Claude Code process exited with code 127'), { - errorClass: 'process_exited_nonzero', - exitCode: 127 - }) - ), - null - ) - const result = adapter - .open({ - sessionId: 'session-1', +it.each(['the SDK reports the exit first', "Orca's own exit arrives first"])( + "reads a missing CLI from Orca's exit when %s", + async (order) => { + const claude = fakeClaude({ holdInit: true }) + const adapter = adapterAtPublishFor(claude) + const acquired = await adapter.acquire({ + identity: identityFor(), fence: 7, - acquisitionGeneration: acquired.acquisitionGeneration, - signal: new AbortController().signal + spawnToken: 'missing-cli' }) - .catch((error) => error) - await tick() - exited.resolve( - withMissingProviderExecutable( + const connection = claude.connections[0] + const exited = Promise.withResolvers() + connection.exited = exited.promise + const report = Promise.withResolvers() + connection.initializationResult = () => + runClaudeControl('initialize', () => report.promise, null) + // A supervised spawn ENOENT: the supervisor exits 127 and the SDK reports only that exit. + const sdkReport = Object.assign(new Error('Claude Code process exited with code 127'), { + errorClass: 'process_exited_nonzero', + exitCode: 127 + }) + const ownExit = withMissingProviderExecutable( withObservedProviderExit( new Error('claude stream-json exited (code 127): spawn claude ENOENT') ) ) - ) - const error = await result - expect(error).toMatchObject({ retry: 'never', fact: { kind: 'cliMissing' } }) - claude.dispose() - await adapter.closeAll() -}) + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + await tick() + if (order === 'the SDK reports the exit first') { + report.reject(sdkReport) + await tick() + exited.resolve(ownExit) + } else { + exited.resolve(ownExit) + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(ownExit, { expected: false }) + await tick() + report.reject(sdkReport) + } + const error = await result + expect(error).toMatchObject({ retry: 'never', fact: { kind: 'cliMissing' } }) + // The verdict keeps Orca's record, diagnostic and absence marker, not the SDK's summary. + expect(findInErrorChain(error, (cause) => (cause === ownExit ? true : undefined))).toBe(true) + claude.dispose() + await adapter.closeAll() + } +) it('settles an unanswered Claude startup exit with the same bounded recovery eligibility', async () => { vi.useFakeTimers() diff --git a/src/main/claude/structured/claude-structured-session-startup.ts b/src/main/claude/structured/claude-structured-session-startup.ts index 01234501465b..4b0e4024f45c 100644 --- a/src/main/claude/structured/claude-structured-session-startup.ts +++ b/src/main/claude/structured/claude-structured-session-startup.ts @@ -1,4 +1,5 @@ import { isClaudeQueryClosed } from '../claude-agent-sdk-control-requests' +import { claudeStartFailure } from '../claude-startup-failure' import { startupAcquisitionRefusal, providerExitObserved @@ -82,18 +83,20 @@ export async function readClaudeStartupFacts(input: { }) const initialization = await Promise.race([ control.catch(async (error: unknown) => { + let proof: unknown try { input.initProof.seen() } catch (proofError) { - if (startupAcquisitionRefusal(proofError)) { - throw proofError - } - if (isClaudeQueryClosed(error)) { - throw input.connection.startupFailure?.() ?? proofError - } + // A refusal, or what closed the query before Claude identified itself, is an answer. + proof = + startupAcquisitionRefusal(proofError) || isClaudeQueryClosed(error) ? proofError : null } - const ended = await claudeExitForReport(input.connection, error) - throw input.connection.startupFailure?.() ?? ended + throw claudeStartFailure([ + input.connection.startupFailure?.(), + proof, + error, + await claudeExitForReport(input.connection, error) + ]) }), Promise.race([ input.initProof.promise.then(() => new Promise(() => {})), diff --git a/src/main/codex/codex-structured-start-verdict.ts b/src/main/codex/codex-structured-start-verdict.ts index c6a6e74bcd78..075a57c50f23 100644 --- a/src/main/codex/codex-structured-start-verdict.ts +++ b/src/main/codex/codex-structured-start-verdict.ts @@ -22,7 +22,8 @@ export function openCodexStart( new Error(`no spawned codex child for session ${input.sessionId}`) ) return opening.catch((error: unknown) => { - throw codexStartFailure(error, attempt?.exitProven === true && !attempt.cancelled) + const exited = attempt?.exitProven === true && !attempt.cancelled + throw codexStartFailure([error, exited ? attempt?.exitError : undefined], exited) }) } diff --git a/src/main/jsonl-rpc/agent-connection.ts b/src/main/jsonl-rpc/agent-connection.ts index a5f58a27d36a..ec1849c897b6 100644 --- a/src/main/jsonl-rpc/agent-connection.ts +++ b/src/main/jsonl-rpc/agent-connection.ts @@ -69,6 +69,10 @@ export class JsonlRpcAgentConnection { ...options, // Output often ends just before the exit whose report says how the agent ended. onInputEnd: (finish) => { + // Output reports its end and then its close; both are one end, with one wait. + if (this.inputEnded) { + return + } this.inputEnded = true if (this.closing || managed.rootVerdict === 'exited') { finish() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts index ac3004b112ab..8a5323179a18 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts @@ -20,8 +20,10 @@ import { AgentSessionAcquisitionRootExitObservedError } from './structured-agent-session-acquisition-errors' import { + launchPreparationDiagnosticOf, providerExitObserved, providerStartupFailureFact, + startupAcquisitionRefusal, withObservedProviderExit } from './structured-agent-session-failure-text' @@ -29,8 +31,11 @@ import { export type AgentSessionStartRetry = 'safe' | 'never' /** An adapter's reading of one error in the chain: the agent's answer, final or declared - * temporary, or its own transport ending with no answer once output drained. */ -export type AgentSessionStartAnswer = (cause: Error) => 'final' | 'temporary' | 'exited' | undefined + * temporary; Orca's own record of its transport ending with no answer once output drained; or + * an SDK's report of that ending, which tells less than Orca's record of it. */ +export type AgentSessionStartAnswer = ( + cause: Error +) => 'final' | 'temporary' | 'exited' | 'reported' | undefined export class AgentSessionStartFailure extends Error { readonly fact: SubmissionRejectionFact @@ -46,23 +51,52 @@ export class AgentSessionStartFailure extends Error { } } -/** `exited`: the adapter saw its process end on its own, which words even a final answer as the - * agent's own failure to start without changing whether it may run again. */ +/** One verdict from every observation an adapter kept of a failed start: the operation's errors, + * a failed result seen for it and Orca's own record of its process ending, earliest first. It + * may run again only if every observation allows it; the most specific one explains it, ties + * going to the earliest. `exited`: the adapter saw its process end on its own, which words even + * a final answer as the agent's own failure to start without changing whether it may run again. */ export function agentSessionStartFailure( - error: unknown, + evidence: unknown, answer?: AgentSessionStartAnswer, exited = false ): AgentSessionStartFailure { - if (error instanceof AgentSessionStartFailure) { - return error + const observed = Array.isArray(evidence) + ? evidence.filter((cause, index) => cause != null && evidence.indexOf(cause) === index) + : [evidence] + const verdict = observed.find((cause) => cause instanceof AgentSessionStartFailure) + if (verdict) { + return verdict } - const disposition = startDisposition(error, answer) + const readings = observed.map((cause) => startDisposition(cause, answer)) + const ranks = observed.map((cause, index) => explanationRank(cause, readings[index].answers)) + const selected = ranks.indexOf(Math.min(...ranks)) + const error = observed[selected] // An adapter's own exit error words the start as the agent's failure, as an observed exit does. const worded = - (exited || disposition.exited) && error instanceof Error + (exited || readings[selected].exited) && error instanceof Error ? withObservedProviderExit(new Error(error.message, { cause: error })) : error - return new AgentSessionStartFailure(error, disposition.retry, providerStartupFailureFact(worded)) + const retry = readings.every((reading) => reading.retry === 'safe') ? 'safe' : 'never' + return new AgentSessionStartFailure(error, retry, providerStartupFailureFact(worded)) +} + +/** Most specific first: what the user can act on, the request's deadline, the agent's answer, + * Orca's record of its exit, an SDK's report of that exit, then anything else. */ +function explanationRank(cause: unknown, answers: ReadonlySet): number { + const rank = [ + agentCliInstallationProblemOf(cause), + startupAcquisitionRefusal(cause), + providerExecutableMissing(cause), + providerLaunchFolderMissing(cause), + launchPreparationDiagnosticOf(cause) ?? argumentProblemOf(cause), + providerRequestTimeoutOf(cause), + ['final', 'temporary', 'never', 'safe'].some((read) => answers.has(read)) || + providerResourcePressure(cause), + answers.has('exited') || providerExitObserved(cause), + answers.has('reported') + ].findIndex(Boolean) + return rank === -1 ? 9 : rank } /** The adapter's verdict on an error where it read an answer or its own exit; otherwise none. */ @@ -167,7 +201,17 @@ export function retryableAgentSessionAcquisition(error: unknown): boolean { function startDisposition( error: unknown, answer?: AgentSessionStartAnswer -): { retry: AgentSessionStartRetry; exited: boolean } { +): { + retry: AgentSessionStartRetry + exited: boolean + answers: Set | AgentSessionStartRetry> +} { + const answers = new Set | AgentSessionStartRetry>() + findInErrorChain(error, (cause) => { + answers.add(cause instanceof AgentSessionStartFailure ? cause.retry : answer?.(cause)) + // An adapter's verdict already covers the errors beneath it. + return cause instanceof AgentSessionStartFailure ? true : undefined + }) if ( providerRequestTimeoutOf(error) || argumentProblemOf(error) || @@ -181,27 +225,20 @@ function startDisposition( isAgentSessionPreSpawnError(cause) ? true : undefined - ) + ) || + answers.has('never') || + answers.has('final') ) { - return { retry: 'never', exited: false } - } - const answers = new Set | AgentSessionStartRetry>() - findInErrorChain(error, (cause) => { - answers.add(cause instanceof AgentSessionStartFailure ? cause.retry : answer?.(cause)) - // An adapter's verdict already covers the errors beneath it. - return cause instanceof AgentSessionStartFailure ? true : undefined - }) - if (answers.has('never') || answers.has('final')) { - return { retry: 'never', exited: false } + return { retry: 'never', exited: false, answers } } - const exited = answers.has('exited') + const exited = answers.has('exited') || answers.has('reported') const safe = exited || answers.has('safe') || answers.has('temporary') || providerExitObserved(error) || providerResourcePressure(error) - return { retry: safe ? 'safe' : 'never', exited } + return { retry: safe ? 'safe' : 'never', exited, answers } } /** The host ran short of processes or files; the same launch can run once that passes. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts index 7fcae2044bb3..df84c31604a2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts @@ -10,6 +10,18 @@ import { readAntigravitySignInLine } from '../../antigravity/antigravity-sign-in import { findInErrorChain } from '../../provider-process/provider-error-chain' import { createFakePipedChild } from '../../../shared/__fixtures__/fake-spawned-child' import { closeProviderTimelineRigs } from '../agent-session-timeline/provider-timeline-assembler-test-support' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session/lifecycle/agent-session-mutation-envelope' +import { + adapterAtPublishFor, + fakeClaude, + identityFor, + tick +} from '../../claude/structured/claude-structured-session-test-support' +import { + sessionId as piSessionId, + setup as setupPi +} from '../../pi/rpc-session-adapter.test-fixture' +import { withObservedProviderExit } from './structured-agent-session-failure-text' import { AgentSessionStartFailure } from './structured-agent-session-start-failure' import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' @@ -33,6 +45,7 @@ import { HOST_TEST_NOW as NOW, HOST_TEST_SESSION as SESSION, hostTestAttachParams, + hostTestOperationId, resetHostTestOperationIds } from './structured-agent-session-host-test-data' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' @@ -70,13 +83,14 @@ vi.mock('./structured-agent-session-exit-observation', async (original) => { const CALLER = { callerKey: 'client-1' } const CEILING_MS = 20_000 -type Cycle = 'ready' | 'final' | 'retried' | 'expired' -const CYCLES: readonly Cycle[] = ['ready', 'final', 'retried', 'expired'] +type Cycle = 'ready' | 'final' | 'retried' | 'expired' | 'stopped' +const CYCLES: readonly Cycle[] = ['ready', 'final', 'retried', 'expired', 'stopped'] const SCRIPTS: Record = { ready: ['ok'], final: ['refuse'], retried: ['hang', 'ok'], - expired: ['hang'] + expired: ['hang'], + stopped: ['hang'] } let root: string @@ -185,7 +199,10 @@ async function runStarts(count: number) { bufferedExits: runtimeState.startFlights['exits'].size, receipts: retainedReceipts(), rootExitListeners: created.rootExits.reduce((sum, exit) => sum + exit['listeners'].size, 0), - liveChildren: codex.connections.filter((connection) => !connection.closed).length + liveChildren: codex.connections.filter((connection) => !connection.closed).length, + // Host work still tracked or queued after its caller returned. + trackedTasks: host!['tasks']['attaching'].size, + queuedTasks: host!['tasks']['chains'].size }) const eventually = (assertion: () => unknown) => vi.waitFor(assertion, { timeout: 10_000, interval: 0 }) @@ -222,9 +239,12 @@ async function runStarts(count: number) { ready: [], final: [], retried: [], - expired: [] + expired: [], + stopped: [] } let rows = 0 + // What a Stopped start holds while its child's cleanup is still running. + const stopping: ReturnType[] = [] // What one start holds while its child is still answering. const starting: ReturnType[] = [] for (let round = 0; round < count; round++) { @@ -249,6 +269,33 @@ async function runStarts(count: number) { await eventually(() => expect(answers).toEqual([])) await vi.advanceTimersByTimeAsync(CEILING_MS) } + if (cycle === 'stopped') { + await eventually(() => expect(answers).toEqual([])) + // The user stops the start while its open waits; the child's cleanup is held, then ends. + const connection = codex.connections.at(-1)! + const close = connection.close + const cleanup = Promise.withResolvers() + connection.close = async () => { + await cleanup.promise + return close() + } + const fence = store.getRecord(SESSION)!.lease.runtimeFence + const stop = { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: SESSION, + fields: {} + }) + } + const stopped = host.cancel(CALLER, { envelope: stop }) + await vi.advanceTimersByTimeAsync(100) + stopping.push(held()) + cleanup.resolve() + expect(await stopped).toMatchObject({ ok: true }) + } // The start's owner decides it: the message runs on a ready child, or one row says why not. const started = cycle === 'ready' || cycle === 'retried' await eventually(async () => { @@ -258,7 +305,7 @@ async function runStarts(count: number) { expect((await submission(send.envelope.clientOperationId))?.dispatchState).toBe( started ? 'accepted' : 'rejected' ) - expect(await failureRows()).toBe(rows + (started ? 0 : 1)) + expect(await failureRows()).toBe(rows + (cycle === 'final' || cycle === 'expired' ? 1 : 0)) }) expect(codex.connections).toHaveLength(spawned + (cycle === 'retried' ? 2 : 1)) await host.flushStreamedEvents(SESSION) @@ -274,7 +321,7 @@ async function runStarts(count: number) { const counts = Object.fromEntries( Object.entries(done).map(([field, count]) => [field, count - begun[field]]) ) - return { after, starting, counts } + return { after, starting, stopping, counts } } it('returns every start resource to baseline and does constant work per start, at N and 10N', async () => { @@ -283,7 +330,8 @@ it('returns every start resource to baseline and does constant work per start, a for (const cycle of CYCLES) { const [first] = n.after[cycle] // Nothing a finished start owned survives it: no owner, abort handle, tracked attempt, flight, - // buffered exit, adapter receipt or root-exit listener, and no exited child left running. + // buffered exit, adapter receipt, root-exit listener or tracked task, and no exited child left + // running. expect(first, cycle).toMatchObject({ abortHandles: 0, trackedStarts: 0, @@ -292,7 +340,9 @@ it('returns every start resource to baseline and does constant work per start, a bufferedExits: 0, receipts: 0, rootExitListeners: 0, - liveChildren: cycle === 'ready' || cycle === 'retried' ? 1 : 0 + liveChildren: cycle === 'ready' || cycle === 'retried' ? 1 : 0, + trackedTasks: 0, + queuedTasks: 0 }) // Timers and abort listeners held after each start do not grow with the starts before it. for (const held of tenN.after[cycle]) { @@ -314,16 +364,31 @@ it('returns every start resource to baseline and does constant work per start, a for (const held of tenN.starting) { expect(held).toEqual(n.starting[0]) } - // Per round of four starts: five spawns and opens (the retried start opens twice) and a failure - // row for the final and the expired start. Router reads and retirements reach the registered - // adapters a fixed number of times per open, whatever the starts before it. + // A Stop releases the start's owner at once; while its child's cleanup is held, that cleanup is + // the one task still tracked and queued, however many Stops came before it. + expect(n.stopping[0]).toMatchObject({ + owners: 0, + flights: 0, + abortHandles: 0, + receipts: 0, + trackedTasks: 1, + queuedTasks: 1, + liveChildren: 1 + }) + for (const held of tenN.stopping) { + expect(held).toEqual(n.stopping[0]) + } + // Per round of five starts: six spawns and opens (the retried start opens twice) and a failure + // row for the final and the expired start; a Stopped start reads no ending. Router reads and + // retirements reach the registered adapters a fixed number of times per open, whatever the + // starts before it. expect(n.counts).toMatchObject({ - spawns: 5, - opens: 5, + spawns: 6, + opens: 6, codexEndingReads: 4, otherEndingReads: 4, - codexRetirements: 5, - otherRetirements: 5, + codexRetirements: 6, + otherRetirements: 6, failureRows: 2 }) for (const [field, count] of Object.entries(n.counts)) { @@ -397,3 +462,129 @@ it('ACP: each refused start keeps its own cause and leaves no receipt or child, expect(retained).toEqual(Array(count).fill(0)) } }) + +/** What a real Claude or Pi adapter keeps from `count` starts of each ending, retired by the + * start's owner as the host does: a ready start, and one whose agent exited before its open. */ +async function adapterStarts( + agent: 'claude' | 'pi', + count: number +): Promise<{ left: Record[]; spawns: number }> { + const harness = agent === 'claude' ? await claudeStarts() : await piStarts() + const left: Record[] = [] + try { + for (let round = 0; round < count; round++) { + for (const exitsFirst of [false, true]) { + const generation = await harness.acquire() + if (exitsFirst) { + harness.exitChild() + } + const opened = await harness.open(generation).then( + () => 'ready', + (error: unknown) => error + ) + if (exitsFirst) { + // The open answers with the ending the exit left, a verdict the owner can act on. + expect(opened).toBeInstanceOf(AgentSessionStartFailure) + } else { + expect(opened).toBe('ready') + } + harness.adapter.retireStartup?.(harness.sessionId, generation) + // Retired by its owner, even while a ready start's session stays live. + const receipts = retainedReceipts() + await harness.adapter.closeSession(harness.sessionId) + await harness.settle() + left.push({ receipts, ...harness.held() }) + } + } + return { left, spawns: harness.spawns() } + } finally { + await harness.adapter.closeAll() + await closeProviderTimelineRigs() + } +} + +async function claudeStarts() { + const claude = fakeClaude() + const adapter = adapterAtPublishFor(claude) + const sessionId = identityFor().sessionId + return { + adapter, + sessionId, + acquire: async () => + (await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'perf' })) + .acquisitionGeneration, + open: (acquisitionGeneration: string) => + adapter.open({ + sessionId, + fence: 7, + acquisitionGeneration, + signal: new AbortController().signal + }), + exitChild: () => { + const connection = claude.connections.at(-1)! + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(withObservedProviderExit(new Error('claude exited')), { + expected: false + }) + }, + settle: async () => { + adapter.acknowledgeSessionRelease?.(sessionId) + await tick() + }, + held: () => ({ + sessions: adapter['sessions'].size, + acquisitions: adapter['acquisitions']['attempts'].size, + exits: adapter['exits'].size + }), + spawns: () => claude.connections.length + } +} + +async function piStarts() { + const pi = await setupPi() + const { adapter, input, connections } = pi + await adapter.closeSession(piSessionId) + return { + adapter, + sessionId: piSessionId, + acquire: async () => (await adapter.acquire(input)).acquisitionGeneration, + open: (acquisitionGeneration: string) => + adapter.open({ + sessionId: piSessionId, + fence: 7, + acquisitionGeneration, + signal: new AbortController().signal + }), + exitChild: () => connections.at(-1)!.exit(), + settle: async () => { + await adapter.drainObservedExits() + adapter.acknowledgeSessionRelease(piSessionId) + }, + held: () => ({ + sessions: adapter['sessions'].size, + rootExitListeners: connections.reduce( + (sum, connection) => sum + connection['exitListeners'].length, + 0 + ) + }), + spawns: () => connections.length + } +} + +it.each(['claude', 'pi'] as const)( + '%s: each start retires its receipt and exit subscription, at N and 10N', + async (agent) => { + vi.useRealTimers() + const n = await adapterStarts(agent, 10) + const tenN = await adapterStarts(agent, 100) + const settled = + agent === 'claude' + ? { sessions: 0, acquisitions: 0, exits: 0 } + : { sessions: 0, rootExitListeners: 0 } + for (const held of [...n.left, ...tenN.left]) { + expect(held).toEqual({ receipts: 0, ...settled }) + } + expect(tenN.spawns - (agent === 'pi' ? 1 : 0)).toBe(10 * (n.spawns - (agent === 'pi' ? 1 : 0))) + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts index 86aa14d499d7..f7b2c4a416a3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts @@ -3,7 +3,14 @@ import sdkExit from '../../claude/fixtures/startup-sdk-process-exit.json' import { CodexAppServerUnsupportedError } from '../../codex/app-server/codex-app-server-session' import { PassThrough } from 'node:stream' import { describe, expect, it, vi } from 'vitest' -import { AcpAgentError, AcpInvalidResponseError, AcpRpcError } from '../../acp/acp-errors' +import { + AcpAgentError, + AcpAgentExitedError, + AcpConnectionClosedError, + AcpInvalidResponseError, + AcpRequestTimeoutError, + AcpRpcError +} from '../../acp/acp-errors' import { acpRpcEnvelopeSchema, settleAcpResponse } from '../../acp/acp-rpc-envelope' import { ClaudeControlRequestError, @@ -13,7 +20,14 @@ import { createCodexAppServerRecordDispatcher } from '../../codex/app-server/cod import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' import { JsonlRpcPeer, JsonlRpcResponseError } from '../../jsonl-rpc/peer' import { withProviderRequestTimeout } from '../../provider-process/provider-request-timeout' -import { withProviderSpawnAbsence } from '../../provider-process/provider-executable-missing' +import { + withMissingProviderExecutable, + withProviderSpawnAbsence +} from '../../provider-process/provider-executable-missing' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session/failures/agent-session-failure-words' +import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' +import { JsonlRpcAgentExitError } from '../../jsonl-rpc/agent-connection' import { AgentSessionAcquisitionExitProvenError, AgentSessionAcquisitionExitUnprovenError, @@ -25,12 +39,13 @@ import { agentSessionAcquireFailure, agentSessionStartFailure, retryableAgentSessionAcquisition, - type AgentSessionStartAnswer + type AgentSessionStartAnswer, + type AgentSessionStartFailure } from './structured-agent-session-start-failure' -import { codexStartupAnswer } from '../../codex/app-server/codex-startup-outcome' -import { acpStartupAnswer } from '../../acp/acp-startup-outcome' -import { piStartupAnswer } from '../../pi/rpc-startup-outcome' -import { claudeStartupAnswer } from '../../claude/claude-startup-failure' +import { codexStartFailure, codexStartupAnswer } from '../../codex/app-server/codex-startup-outcome' +import { acpStartFailure, acpStartupAnswer } from '../../acp/acp-startup-outcome' +import { piStartFailure, piStartupAnswer } from '../../pi/rpc-startup-outcome' +import { claudeStartFailure, claudeStartupAnswer } from '../../claude/claude-startup-failure' import captures from './__fixtures__/startup-protocol-rejections.json' // Each adapter reads only its own answers; the shared rule then gives the generic verdict. @@ -352,3 +367,171 @@ describe('positive provider startup retry contracts', () => { expectExitEvidenceRequired(error) }) }) + +describe('one verdict from every observation of a failed start', () => { + const recordedSdkExit = async (exitCode: number) => { + const recorded = sdkExit.records[0].initializationError + if (!recorded) { + throw new Error('missing recorded initialization error') + } + // The Debian lane's supervised spawn ENOENT: the recorded SDK shape, with the supervisor's 127. + const cause = Object.assign(new Error(`Claude Code process exited with code ${exitCode}`), { + ...recorded, + exitCode + }) + return runClaudeControl('initialize', () => Promise.reject(cause)).catch( + (error: unknown) => error + ) + } + const claudeOwnExit = (message: string) => withObservedProviderExit(new Error(message)) + const recordedCodexRefusal = async () => { + const pending = Promise.withResolvers() + const failed = pending.promise.catch((caught: unknown) => caught) + const dispatcher = createCodexAppServerRecordDispatcher({ + handlers: {}, + writeResponse: vi.fn(), + onProtocolFailure: vi.fn() + }) + dispatcher.addPending(captures.codex.request.id, { + method: captures.codex.request.method, + ...pending, + timer: undefined + }) + dispatcher.dispatch(captures.codex.response) + return failed + } + const loadTimeout = new AcpRequestTimeoutError('session/load', 300_000) + type Row = { + name: string + verdict: (evidence: unknown[]) => AgentSessionStartFailure + observations: () => Promise<[explains: unknown, other: unknown]> + retry: 'safe' | 'never' + kind: string + } + const rows: Row[] = [ + { + name: "Claude: Orca's exit that found no CLI over the SDK's exit 127 report", + verdict: (evidence) => claudeStartFailure(evidence), + observations: async () => [ + withMissingProviderExecutable( + claudeOwnExit('claude stream-json exited (code 127): spawn claude ENOENT') + ), + await recordedSdkExit(127) + ], + retry: 'never', + kind: 'cliMissing' + }, + { + name: "Claude: Orca's plain exit record over the SDK's report of it", + verdict: (evidence) => claudeStartFailure(evidence), + observations: async () => [ + claudeOwnExit('claude stream-json exited (code 1)'), + await recordedSdkExit(1) + ], + retry: 'safe', + kind: 'providerStartFailed' + }, + { + name: 'Claude: a recorded control rejection over its exit', + verdict: (evidence) => claudeStartFailure(evidence), + observations: async () => [ + await runClaudeControl(captures.claude.request.request.subtype, () => + Promise.reject(new Error(captures.claude.response.response.error)) + ).catch((caught: unknown) => caught), + claudeOwnExit('claude stream-json exited (code 1)') + ], + retry: 'never', + kind: 'startFailed' + }, + { + name: 'Codex: a recorded refusal over its drained exit, worded as that exit', + verdict: (evidence) => codexStartFailure(evidence, true), + observations: async () => [ + await recordedCodexRefusal(), + new CodexAppServerExitError('codex app-server exited') + ], + retry: 'never', + kind: 'providerStartFailed' + }, + { + name: 'Codex: a declared temporary answer beside its drained exit', + verdict: (evidence) => codexStartFailure(evidence, true), + observations: async () => [ + new CodexAppServerRequestError('initialize', -32001, 'Server overloaded; retry later.'), + new CodexAppServerExitError('codex app-server exited') + ], + retry: 'safe', + kind: 'providerStartFailed' + }, + { + name: 'ACP: a load deadline over the exit seen while its output drained', + verdict: (evidence) => acpStartFailure(evidence, true), + observations: async () => [loadTimeout, new AcpAgentExitedError('agent exited')], + retry: 'never', + kind: 'startFailed' + }, + { + name: "ACP: a deleted launch folder over the transport's closed summary", + verdict: (evidence) => acpStartFailure(evidence), + observations: async () => [ + withProviderSpawnAbsence(new AcpAgentExitedError('agent exited'), 'launchFolder'), + new AcpConnectionClosedError() + ], + retry: 'never', + kind: 'launchFolderMissing' + }, + { + name: 'Pi: a recorded rejection over its exit', + verdict: (evidence) => piStartFailure(evidence), + observations: async () => [ + new JsonlRpcResponseError(captures.pi.response.command, captures.pi.response.error), + new JsonlRpcAgentExitError('Pi exited while starting') + ], + retry: 'never', + kind: 'startFailed' + }, + { + name: 'Pi: its exit explains an unexplained failure, which still keeps the start final', + verdict: (evidence) => piStartFailure(evidence), + observations: async () => [ + new JsonlRpcAgentExitError('Pi exited while starting'), + new Error('Invalid agent RPC response') + ], + retry: 'never', + kind: 'providerStartFailed' + } + ] + + it.each(rows)('$name', async ({ verdict, observations, retry, kind }) => { + const [explains, other] = await observations() + // Precedence is the policy's, not the order the adapter saw them in. + for (const evidence of [ + [explains, other], + [other, explains] + ]) { + const failure = verdict(evidence) + expect(failure.cause).toBe(explains) + expect(failure.retry).toBe(retry) + expect(failure.fact.kind).toBe(kind) + } + }) + + it("keeps a deadline's method, duration and words over the exit around it", () => { + const failure = acpStartFailure([new AcpAgentExitedError('agent exited'), loadTimeout], true) + const timedOut = agentSessionFailureFact('startFailed', { + requestTimeout: { method: 'session/load', timeoutMs: 300_000 } + }) + expect(failure.fact).toEqual(timedOut) + expect(agentSessionFailureWords(failure.fact, { surface: 'rejection' })).toEqual( + agentSessionFailureWords(timedOut, { surface: 'rejection' }) + ) + }) + + it('still lets an unanswered exit run again when every observation is that exit', async () => { + const own = claudeOwnExit('claude stream-json exited (code 1)') + expect(claudeStartFailure([own, await recordedSdkExit(1)]).retry).toBe('safe') + expect(piStartFailure([new JsonlRpcAgentExitError('Pi exited while starting')]).retry).toBe( + 'safe' + ) + }) +}) diff --git a/src/main/pi/rpc-startup-open.ts b/src/main/pi/rpc-startup-open.ts index b3bdbf420437..2be19aa7c091 100644 --- a/src/main/pi/rpc-startup-open.ts +++ b/src/main/pi/rpc-startup-open.ts @@ -5,11 +5,12 @@ import type { import type { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' import type { StructuredAgentSessionProviderOpenInput } from '../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' import { JsonlRpcAgentExitError } from '../jsonl-rpc/agent-connection' +import { withProviderSpawnAbsence } from '../provider-process/provider-executable-missing' import type { PiRpcSession } from './rpc-session' import { piStartFailure } from './rpc-startup-outcome' -/** Takes this child's own exit when first seen, so a start settling meanwhile cannot miss it; a - * protocol failure before it stays the cause. */ +/** Takes this child's own exit when first seen, so a start settling meanwhile cannot miss it; the + * verdict weighs a failure before it against that exit. */ export function watchPiStartExit( session: PiRpcSession, input: Pick, @@ -20,8 +21,11 @@ export function watchPiStartExit( session.connection.onExit((requested) => { if (!requested) { input.events?.observeRootExit?.() - const exited = new JsonlRpcAgentExitError('Pi exited while starting') - endings.record(sessionId, session.generation, piStartFailure(session.failure ?? exited)) + const exited = withProviderSpawnAbsence( + new JsonlRpcAgentExitError('Pi exited while starting'), + session.connection.spawnAbsence + ) + endings.record(sessionId, session.generation, piStartFailure([session.failure, exited])) } }) } @@ -46,7 +50,11 @@ export async function openPiStart( ) } if (session.connection.closed) { - throw session.failure ?? new Error('Pi child ended before it opened') + throw ( + endings.get(input.sessionId, input.acquisitionGeneration) ?? + session.failure ?? + new Error('Pi child ended before it opened') + ) } return await session.open(input.signal) } catch (error) { diff --git a/src/main/provider-process/provider-exit-drain-scaling.test.ts b/src/main/provider-process/provider-exit-drain-scaling.test.ts index afc92cfd274b..4d1014224744 100644 --- a/src/main/provider-process/provider-exit-drain-scaling.test.ts +++ b/src/main/provider-process/provider-exit-drain-scaling.test.ts @@ -224,3 +224,70 @@ it.each(Object.keys(TRANSPORTS))( expect(tenN.drains).toBe(10 * n.drains) } ) + +/** JSON-lines output ending while its root still runs, then the exit after it, no exit at all, or + * Orca closing while an exit's drain is pending; counts what each cycle leaves at its close. */ +async function jsonlLiveRootCycles(count: number) { + drains.length = 0 + const timers = vi.spyOn(globalThis, 'setTimeout') + const settled = { answered: 0, failed: 0 } + const left: { timers: number; drainListeners: number }[] = [] + const waitsAfterExit: number[] = [] + for (let cycle = 0; cycle < count; cycle++) { + for (const ending of ['exit after EOF', 'no exit', 'close mid-drain'] as const) { + const opened = await TRANSPORTS.jsonl() + const { child } = opened + opened.pending.then( + () => (settled.answered += 1), + () => (settled.failed += 1) + ) + if (ending === 'close mid-drain') { + child.emit('exit', 1, null) + } else { + child.stdout.end() + await vi.advanceTimersByTimeAsync(0) + if (ending === 'exit after EOF') { + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS / 2) + child.emit('exit', 1, null) + await vi.advanceTimersByTimeAsync(0) + // The exit settles the call and ends the wait its output's end started. + waitsAfterExit.push(vi.getTimerCount()) + } else { + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS) + } + } + await vi.advanceTimersByTimeAsync(0) + const closing = opened.close() + if (ending === 'no exit') { + child.emit('exit', 0, null) + } + await closing + await vi.advanceTimersByTimeAsync(0) + left.push({ timers: vi.getTimerCount(), drainListeners: drainListeners() }) + child.emit('close', 0, null) + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS) + } + } + const timersScheduled = timers.mock.calls.length + timers.mockRestore() + const finished = drains.map((drain) => drain.finished) + return { settled, left, waitsAfterExit, timersScheduled, drains: drains.length, finished } +} + +it('jsonl: output ending before a live root exits leaves no timer or listener, at N and 10N', async () => { + vi.useFakeTimers() + const n = await jsonlLiveRootCycles(10) + const tenN = await jsonlLiveRootCycles(100) + for (const run of [n, tenN]) { + for (const held of run.left) { + expect(held).toEqual({ timers: 0, drainListeners: 0 }) + } + expect(run.waitsAfterExit.every((waits) => waits === 0)).toBe(true) + expect(run.finished.every((finished) => finished === 1)).toBe(true) + } + // Every unanswered call fails exactly once, however its output and process ended. + expect(n.settled).toEqual({ answered: 0, failed: 30 }) + expect(tenN.settled).toEqual({ answered: 0, failed: 300 }) + expect(tenN.timersScheduled).toBe(10 * n.timersScheduled) + expect(tenN.drains).toBe(10 * n.drains) +}) From eeac0fc0242002ac27565d8bbf7e6b974203f620 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:36:25 -0800 Subject: [PATCH 21/21] fix(acp): keep a missed saved-session load deadline instead of opening a replacement A live agent that let session/load run past its deadline fell through to the takeover path, and the replacement session/new answer then replaced the deadline the user had to see. The takeover predicate now propagates any provider request deadline unchanged, read through the existing typed timeout. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../acp/acp-session-reopen-failure.test.ts | 26 ++++++++++++++----- src/main/acp/acp-session-reopen-failure.ts | 11 ++++++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/main/acp/acp-session-reopen-failure.test.ts b/src/main/acp/acp-session-reopen-failure.test.ts index 25acf5cae460..1a75b3ec7471 100644 --- a/src/main/acp/acp-session-reopen-failure.test.ts +++ b/src/main/acp/acp-session-reopen-failure.test.ts @@ -62,7 +62,10 @@ it('propagates a temporary load envelope before takeover or a dead-child asserti await rig.adapter.closeAll() }) -it("keeps a load deadline's words when the agent exited while its output drained", async () => { +it.each([ + { agent: 'exited while its output drained', exits: true }, + { agent: 'is still running', exits: false } +])("keeps a load deadline's words when the agent $agent", async ({ exits }) => { vi.useFakeTimers() const spawned: ReturnType[] = [] const agents: AcpScriptedAgent[] = [] @@ -81,8 +84,9 @@ it("keeps a load deadline's words when the agent exited while its output drained agent.on('initialize', (frame) => agent.reply(frame, { protocolVersion: 1, agentCapabilities: { loadSession: true } }) ) - // The agent never answers the load. + // The agent never answers the load, but would open a replacement session. agent.on('session/load', () => {}) + agent.on('session/new', (frame) => agent.reply(frame, { sessionId: 'replacement-id' })) const child = Object.assign(createFakePipedChild(), { pid: 9_999_999, stdout: agent.stdout, @@ -90,6 +94,12 @@ it("keeps a load deadline's words when the agent exited while its output drained stderr: new PassThrough(), kill: () => true }) + // A live agent exits when Orca closes its input. + agent.stdin.once('finish', () => { + if (!exits) { + child.emit('exit', 0, null) + } + }) spawned.push(child) agents.push(agent) return createAcpAgentConnection(launch, options, () => child) @@ -99,10 +109,14 @@ it("keeps a load deadline's words when the agent exited while its output drained try { const acquired = rig.acquire() await vi.advanceTimersByTimeAsync(299_500) - // The agent exits unanswered at 299.5 s, holding stdout open; the load's own deadline fires at - // 300 s, inside the output drain that the exit started. - spawned[0].emit('exit', 1, null) + // An exiting agent leaves unanswered at 299.5 s, holding stdout open; the load's own deadline + // fires at 300 s, inside the output drain that the exit started. + if (exits) { + spawned[0].emit('exit', 1, null) + } await vi.advanceTimersByTimeAsync(500 + PROVIDER_EXIT_OUTPUT_DRAIN_MS) + // No replacement session is opened, even in a live agent that would answer one. + expect(agents[0].frames.map((frame) => frame.method)).not.toContain('session/new') await acquired const timedOut = agentSessionFailureFact('startFailed', { requestTimeout: { method: 'session/load', timeoutMs: 300_000 } @@ -112,8 +126,6 @@ it("keeps a load deadline's words when the agent exited while its output drained expect(agentSessionFailureWords(rig.openVerdict()!.fact, { surface: 'rejection' })).toEqual( agentSessionFailureWords(timedOut, { surface: 'rejection' }) ) - // No replacement session is opened in an agent that already exited. - expect(agents[0].frames.map((frame) => frame.method)).not.toContain('session/new') expect(spawned).toHaveLength(1) } finally { await rig.adapter.closeAll() diff --git a/src/main/acp/acp-session-reopen-failure.ts b/src/main/acp/acp-session-reopen-failure.ts index bc515b2b1f49..6e653694eacb 100644 --- a/src/main/acp/acp-session-reopen-failure.ts +++ b/src/main/acp/acp-session-reopen-failure.ts @@ -4,6 +4,7 @@ import type { AgentSessionProviderHandleLink } from '../../shared/agent-session/ import type { ProviderTimelineEvent } from '../native-chat/agent-session-timeline/provider-timeline-event' import { AcpAuthRequiredError, AcpRpcError } from './acp-errors' import type { AcpStructuredLaunch } from './acp-structured-launch-resolution' +import { providerRequestTimeoutOf } from '../provider-process/provider-request-timeout' import { temporaryAcpStartupAnswer } from './acp-startup-outcome' @@ -15,8 +16,14 @@ export function acpReopenTakeover( resume: NonNullable, start: { over: boolean; now: number; warn: (fields: { scope: string; error: unknown }) => void } ): Pick { - // A replacement session cannot resolve sign-in failure or an abandoned start. - if (error instanceof AcpAuthRequiredError || temporaryAcpStartupAnswer(error) || start.over) { + // A replacement session cannot resolve sign-in failure, a missed deadline or an abandoned start; + // its own answer would replace the failure the user must see. + if ( + error instanceof AcpAuthRequiredError || + temporaryAcpStartupAnswer(error) || + providerRequestTimeoutOf(error) || + start.over + ) { throw error } if (