diff --git a/src/main/acp/acp-agent-connection.test.ts b/src/main/acp/acp-agent-connection.test.ts index 7e0bccfb3f69..136770f35f85 100644 --- a/src/main/acp/acp-agent-connection.test.ts +++ b/src/main/acp/acp-agent-connection.test.ts @@ -11,7 +11,9 @@ import { type AcpAgentConnection, type AcpAgentConnectionOptions } from './acp-agent-connection' -import { AcpAuthRequiredError } from './acp-errors' +import { AcpAgentExitedError, AcpAuthRequiredError } from './acp-errors' +import { acpStartFailure } from './acp-startup-outcome' +import captures from '../native-chat/agent-session-wire/__fixtures__/startup-protocol-rejections.json' import { PROVIDER_SPAWN_FAILURE_MARKER } from '../provider-process/provider-spawn-failure-report' import { providerExecutableMissing } from '../provider-process/provider-executable-missing' import { AcpScriptedAgent, deferred, tick } from './acp-scripted-agent.test-support' @@ -236,6 +238,8 @@ describe('ACP process-owning connection', () => { const rejected = expect(connection.prompt([...prompt])).rejects.toThrow('broken pipe') child.stdin.emit('error', new Error('broken pipe')) child.emit('close', 0, null) + // The call waits the drain bound for an answer the agent may already have written. + await vi.advanceTimersByTimeAsync(1_000) await rejected expect(connection.closed).toBe(true) expect(connection.rootVerdict).toBe('live') @@ -307,6 +311,33 @@ describe('ACP process-owning connection', () => { } ) + it('keeps the final answer an agent wrote just before it exited, and only then reads its exit', async () => { + const { response } = captures.acp + const onExit = vi.fn() + const { connection, agent, child } = fixture({ onExit }, { exitOnEnd: false }) + // The recorded refusal answers this start's session request just after the agent exited. + agent.on('session/new', (frame) => { + child.emit('exit', 0, null) + setImmediate(() => agent.send({ ...response, jsonrpc: '2.0', id: frame.id })) + }) + const loaded = connection.start(start).catch((error: unknown) => error) + expect(await loaded).toMatchObject({ code: response.error.code, data: response.error.data }) + expect(acpStartFailure(await loaded).retry).toBe('never') + agent.stdout.end() + await vi.waitFor(() => expect(onExit).toHaveBeenCalledOnce()) + }) + + it('answers an unanswered call with the drained exit, which may start once more', async () => { + const { connection, agent, child } = fixture({}, { exitOnEnd: false }) + agent.on('session/new', () => { + child.emit('exit', 1, null) + agent.stdout.end() + }) + const loaded = connection.start(start).catch((error: unknown) => error) + expect(await loaded).toBeInstanceOf(AcpAgentExitedError) + expect(acpStartFailure(await loaded).retry).toBe('safe') + }) + it('isolates early and late exit subscribers and continues delivering exit', () => { const onDiagnostic = vi.fn(() => { throw new Error('diagnostic') @@ -319,11 +350,11 @@ describe('ACP process-owning connection', () => { connection.onExit(failing) connection.onExit(early) expect(() => child.emit('exit', 0, null)).not.toThrow() - expect(early).toHaveBeenCalledExactlyOnceWith({ - code: 0, - signal: null, - processless: false - }) + expect(early).toHaveBeenCalledExactlyOnceWith( + { code: 0, signal: null, processless: false }, + false, + undefined + ) expect(() => connection.onExit(failing)).not.toThrow() const late = vi.fn() connection.onExit(late) diff --git a/src/main/acp/acp-agent-connection.ts b/src/main/acp/acp-agent-connection.ts index 516ce945d91c..2a24ebee24eb 100644 --- a/src/main/acp/acp-agent-connection.ts +++ b/src/main/acp/acp-agent-connection.ts @@ -6,11 +6,15 @@ import { type ProviderProcessExit } from '../provider-process/managed-provider-process' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' +import { + afterProviderOutputDrains, + PROVIDER_EXIT_OUTPUT_DRAIN_MS +} from '../provider-process/provider-exit-output-drain' import { withProviderSpawnAbsence, type ProviderSpawnAbsence } from '../provider-process/provider-executable-missing' -import { AcpConnectionClosedError, AcpSignInPromptError } from './acp-errors' +import { AcpAgentExitedError, AcpConnectionClosedError, AcpSignInPromptError } from './acp-errors' import { resolveAcpPeerOptions, type AcpPeerOptions } from './acp-peer-limits' import { AcpSessionRuntime, type AcpSessionRuntimeOptions } from './acp-session-runtime' @@ -42,9 +46,21 @@ export function createAcpAgentConnection( return new AcpAgentConnection(launch, options, spawnImpl) } +type AcpConnectionLifecycle = { + closing: boolean + error?: Error + /** The process's own failure, such as its spawn; it is no refusal by Orca. */ + processError?: Error + /** Why the connection closed before the process exited, unless Orca closed it or the process + * itself failed. Taken when it closed. */ + lost?: Error + finishExit?: () => void + outputEndTimer?: ReturnType +} + export class AcpAgentConnection extends AcpSessionRuntime { private readonly managed: ManagedProviderProcess - private readonly lifecycle: { closing: boolean; error?: Error } + private readonly lifecycle: AcpConnectionLifecycle private readonly diagnoseLifecycle: (message: string) => void readonly spawned: Promise @@ -86,7 +102,7 @@ export class AcpAgentConnection extends AcpSessionRuntime { } : {}) }) - const lifecycle: { closing: boolean; error?: Error } = { closing: false } + const lifecycle: AcpConnectionLifecycle = { closing: false } const diagnose = (message: string): void => { try { options.onDiagnostic?.(message) @@ -99,7 +115,21 @@ export class AcpAgentConnection extends AcpSessionRuntime { peer, onOtherLine: (line, cut) => signInLine(line, 'stdout', cut), onAuthAnswered: (signedIn) => signIn?.answered?.(signedIn), + // A closed stdin usually means the agent is exiting; its exit settles calls once its output + // drained, so the bound only covers an agent that stopped reading without exiting. + onOutputEnd: (_error, finish) => { + lifecycle.outputEndTimer = setTimeout(finish, PROVIDER_EXIT_OUTPUT_DRAIN_MS) + lifecycle.outputEndTimer.unref() + }, onClose: (error) => { + if ( + !lifecycle.error && + !lifecycle.closing && + managed.rootVerdict !== 'exited' && + error !== lifecycle.processError + ) { + lifecycle.lost = error + } lifecycle.error ??= error if (lifecycle.closing || managed.rootVerdict === 'exited') { return @@ -132,7 +162,10 @@ export class AcpAgentConnection extends AcpSessionRuntime { child.once('spawn', done) child.once('error', done) }) - child.on('error', (error) => super.close(error)) + child.on('error', (error) => { + lifecycle.processError ??= error + super.close(error) + }) signedOut = () => { signIn?.answered?.(false) super.close(new AcpSignInPromptError('The agent asked to sign in in a browser')) @@ -142,16 +175,31 @@ export class AcpAgentConnection extends AcpSessionRuntime { super.close(new AcpConnectionClosedError('The agent printed a sign-in link Orca cannot read')) child.stderr.on('error', (error) => super.close(error)) managed.onExit((exit) => { - const error = withProviderSpawnAbsence( - lifecycle.error ?? - new AcpConnectionClosedError(managed.stderrTail().trim() || `${launch.command} exited`), - managed.spawnAbsence - ) - super.close(error) - try { - options.onExit?.(error, { expected: lifecycle.closing, exit }) - } catch (failure) { - diagnose(`ACP exit listener failed: ${String(failure)}`) + const expected = lifecycle.closing + const finish = (): void => { + clearTimeout(lifecycle.outputEndTimer) + const detail = managed.stderrTail().trim() || `${launch.command} exited` + // Only an exit Orca neither caused nor preceded with its own refusal leaves calls unanswered. + const error = withProviderSpawnAbsence( + lifecycle.error ?? + (expected + ? new AcpConnectionClosedError(detail) + : new AcpAgentExitedError(detail, managed.stderrTail().trim())), + managed.spawnAbsence + ) + super.close(error) + try { + options.onExit?.(error, { expected, exit }) + } catch (failure) { + diagnose(`ACP exit listener failed: ${String(failure)}`) + } + } + // A closed connection has no call left for the agent's output to answer. + if (lifecycle.closing || exit.processless || this.closed) { + finish() + } else { + // An answer written just before exit must still settle its call. + lifecycle.finishExit = afterProviderOutputDrains(child.stdout, finish, child) } }) } @@ -193,10 +241,18 @@ export class AcpAgentConnection extends AcpSessionRuntime { return this.managed.exitStatus } - onExit(listener: (exit: ProviderProcessExit) => void): void { + /** Why the connection closed before the process exited, such as Orca refusing what the agent + * wrote; never the process's own failure or its exit. Kept from the moment it closed. */ + get lostBeforeExit(): Error | undefined { + return this.lifecycle.lost + } + + /** `requested`: Orca was closing the agent when it exited. `lost`: why the connection had + * already closed before the exit, such as Orca refusing what the agent wrote. */ + onExit(listener: (exit: ProviderProcessExit, requested?: boolean, lost?: Error) => void): void { this.managed.onExit((exit) => { try { - listener(exit) + listener(exit, this.lifecycle.closing, this.lifecycle.lost) } catch (failure) { this.diagnoseLifecycle(`ACP exit listener failed: ${String(failure)}`) } @@ -213,6 +269,8 @@ export class AcpAgentConnection extends AcpSessionRuntime { override close(error?: Error): Promise { this.lifecycle.closing ||= this.managed.rootVerdict !== 'exited' + clearTimeout(this.lifecycle.outputEndTimer) + this.lifecycle.finishExit?.() this.drainNotifications(error) return this.managed .close() diff --git a/src/main/acp/acp-errors.ts b/src/main/acp/acp-errors.ts index f8f6a480c3fe..2a863c746af9 100644 --- a/src/main/acp/acp-errors.ts +++ b/src/main/acp/acp-errors.ts @@ -1,5 +1,9 @@ import type { AuthMethod } from './generated/acp-protocol.generated' import { withProviderRequestTimeout } from '../provider-process/provider-request-timeout' +import { + providerDiagnostic, + withProviderDiagnostic +} from '../../shared/agent-session/failures/agent-session-failure' export class AcpRpcError extends Error { constructor( @@ -76,6 +80,18 @@ export class AcpConnectionClosedError extends Error { } } +/** The agent's process ended on its own and its output drained with the call still unanswered. + * Its stderr is the agent's own account, shown behind Details. */ +export class AcpAgentExitedError extends AcpConnectionClosedError { + constructor(message: string, stderr = '') { + super(message) + this.name = 'AcpAgentExitedError' + if (stderr) { + withProviderDiagnostic(this, providerDiagnostic(stderr, 'person')) + } + } +} + export class AcpRequestTimeoutError extends Error { constructor( readonly method: string, diff --git a/src/main/acp/acp-json-rpc-peer.test.ts b/src/main/acp/acp-json-rpc-peer.test.ts index 8d2a78bf0a51..79a419f2ee47 100644 --- a/src/main/acp/acp-json-rpc-peer.test.ts +++ b/src/main/acp/acp-json-rpc-peer.test.ts @@ -9,6 +9,7 @@ import { } from './acp-errors' import { AcpScriptedAgent, deferred, tick } from './acp-scripted-agent.test-support' import { providerRequestTimeoutOf } from '../provider-process/provider-request-timeout' +import captures from '../native-chat/agent-session-wire/__fixtures__/startup-protocol-rejections.json' const peers: AcpJsonRpcPeer[] = [] const agents: AcpScriptedAgent[] = [] @@ -382,4 +383,46 @@ describe('ACP JSON-RPC peer', () => { expect(peer.closed).toBe(true) input.destroy() }) + + it('settles a call with the final answer the agent wrote after it stopped reading', async () => { + const { response } = captures.acp + const { peer, agent } = fixture() + const asked = deferred() + agent.on(captures.acp.request.method, asked.resolve) + const load = peer.request(captures.acp.request.method, {}).catch((error: unknown) => error) + await asked.promise + // The agent closes its stdin before the last chunk of its answer reaches Orca. + agent.stdin.emit('finish') + await expect(peer.request('late', {})).rejects.toBeInstanceOf(AcpConnectionClosedError) + agent.stdout.write(`${JSON.stringify({ ...response, id: 1 })}\n`) + expect(await load).toMatchObject({ code: response.error.code, data: response.error.data }) + expect(peer.closed).toBe(true) + }) + + it('keeps an earlier call answerable when a later write fails', async () => { + const input = new PassThrough() + let writes = 0 + const output = new Writable({ + write(_chunk, _encoding, callback) { + writes += 1 + if (writes === 1) { + callback() + } else { + setImmediate(() => callback(new Error('Broken pipe from write'))) + } + } + }) + const peer = new AcpJsonRpcPeer(input, output) + peers.push(peer) + const first = peer.request('session/new', {}).catch((error: unknown) => error) + await tick() + const second = peer.request('session/set_mode', {}).catch((error: unknown) => error) + await tick() + input.write( + `${JSON.stringify({ jsonrpc: '2.0', id: 1, error: { code: -32600, message: 'refused' } })}\n` + ) + expect(await first).toBeInstanceOf(AcpRpcError) + input.end() + expect(await second).toMatchObject({ message: 'Broken pipe from write' }) + }) }) diff --git a/src/main/acp/acp-json-rpc-peer.ts b/src/main/acp/acp-json-rpc-peer.ts index 4e82096b5082..09defd97acd1 100644 --- a/src/main/acp/acp-json-rpc-peer.ts +++ b/src/main/acp/acp-json-rpc-peer.ts @@ -1,5 +1,4 @@ import type { Readable, Writable } from 'node:stream' -import { z } from 'zod' import { readAcpResponse } from './acp-response-settlement' import type { AcpResponse, AcpRequestOptions, AcpPendingResponse } from './acp-response-settlement' import { @@ -13,12 +12,9 @@ export type { AcpPeerOptions } from './acp-peer-limits' import { settleOversizedAcpLine } from './acp-oversized-lines' import { AcpWriteQueue } from './acp-write-queue' import { detachAcpStreamErrorHandler } from './acp-stdio-error-boundary' -import { - acpRpcEnvelopeSchema, - invalidAcpResponseEnvelope, - settleAcpResponse, - type AcpJsonRpcMessage -} from './acp-rpc-envelope' +import { ProviderOutputEnd } from '../provider-process/provider-output-end' +import { acpRpcEnvelopeSchema, type AcpJsonRpcMessage } from './acp-rpc-envelope' +import { rejectUnreadableAcpReply, settleAcpReply } from './acp-peer-reply' export type { AcpJsonRpcMessage } from './acp-rpc-envelope' export type AcpRequestContext = { id: string | number | null; signal: AbortSignal } export type AcpPeerHandlers = { @@ -27,6 +23,9 @@ export type AcpPeerHandlers = { onNotification?: (method: string, params: unknown) => void onDiagnostic?: (message: string) => void onClose?: (error: Error) => void + /** Bounds how long calls outstanding when the agent stopped reading wait for answers it wrote; + * `finish` settles them. Defaults to the drain of the agent's output. */ + onOutputEnd?: (error: Error, finish: () => void) => void /** Includes non-protocol output and unterminated final lines. */ onOtherLine?: (line: string, cut?: boolean) => void } @@ -37,19 +36,20 @@ export class AcpJsonRpcPeer { private readonly framer: ReturnType private nextId = 1 private terminalError?: Error + private readonly outputEnd: ProviderOutputEnd private drainingNotifications = false private readonly limits: Required constructor( private readonly input: Readable, - private readonly output: Writable, + output: Writable, private readonly handlers: AcpPeerHandlers = {}, options: AcpPeerOptions = {} ) { const limits = resolveAcpPeerOptions(options) this.limits = limits this.writer = new AcpWriteQueue(output, limits.maxQueuedWriteBytes, (error) => - this.close(error) + this.endOutput(error) ) this.incoming = new AcpIncomingRequests( handlers.onRequest, @@ -80,15 +80,15 @@ export class AcpJsonRpcPeer { input.on('end', this.onInputEnd) input.on('close', this.onInputEnd) input.on('error', this.onError) - output.on('close', this.onEnd) - output.on('finish', this.onEnd) - output.on('error', this.onError) - if ( - (this.limits.closeOnInputEnd && (input.destroyed || input.readableEnded)) || - output.destroyed || - !output.writable - ) { + this.outputEnd = new ProviderOutputEnd( + output, + (error) => this.endOutput(error), + () => new AcpConnectionClosedError() + ) + if (this.limits.closeOnInputEnd && (input.destroyed || input.readableEnded)) { this.onEnd() + } else if (output.destroyed || !output.writable) { + this.endOutput(new AcpConnectionClosedError()) } } @@ -107,8 +107,8 @@ export class AcpJsonRpcPeer { params: unknown, options: Partial = {} ): Promise { - if (this.terminalError) { - return Promise.reject(this.terminalError) + if (this.terminalError ?? this.outputEnd.error) { + return Promise.reject(this.terminalError ?? this.outputEnd.error) } if (this.pending.size >= this.limits.maxPendingRequests) { return Promise.reject(new Error('ACP pending request capacity exceeded')) @@ -148,7 +148,8 @@ export class AcpJsonRpcPeer { }) void this.send({ jsonrpc: '2.0', id, method, params }, controller.signal).catch((error) => { const pending = this.pending.get(id) - if (!pending) { + // A write lost to the agent no longer reading leaves the call to the output drain. + if (!pending || this.outputEnd.error) { return } this.pending.delete(id) @@ -191,9 +192,7 @@ export class AcpJsonRpcPeer { if (this.terminalError) { return } - this.output.removeListener('close', this.onEnd) - this.output.removeListener('finish', this.onEnd) - detachAcpStreamErrorHandler(this.output, this.onError) + this.outputEnd.detach() this.terminalError = error this.writer.close(error) for (const pending of this.pending.values()) { @@ -217,13 +216,26 @@ export class AcpJsonRpcPeer { } } private readonly onEnd = (): void => this.close() + + private endOutput(error: Error): void { + if (this.terminalError || !this.outputEnd.begin(error)) { + return + } + this.writer.close(error) + this.outputEnd.settle( + this.pending.size > 0, + this.input, + () => this.close(error), + this.handlers.onOutputEnd && ((finish) => this.handlers.onOutputEnd?.(error, finish)) + ) + } private readonly onInputEnd = (): void => { const tail = this.framer.takeTail() if (tail.trim()) { this.otherLine(tail) } if (this.limits.closeOnInputEnd) { - this.close() + this.close(this.outputEnd.error) } } private readonly onError = (error: Error): void => this.close(error) @@ -266,12 +278,7 @@ export class AcpJsonRpcPeer { const parsed = acpRpcEnvelopeSchema.safeParse(record) if (!parsed.success) { this.diagnose('Ignored invalid ACP JSON-RPC envelope') - const response = z - .object({ id: z.number(), method: z.undefined().optional() }) - .safeParse(record) - if (response.success) { - this.rejectPending(response.data.id, invalidAcpResponseEnvelope(record)) - } + rejectUnreadableAcpReply(record, (id, error) => this.rejectPending(id, error)) return } const frame = parsed.data @@ -294,22 +301,15 @@ export class AcpJsonRpcPeer { } return } - if ('result' in frame === 'error' in frame) { - this.diagnose('Ignored invalid ACP response') - if (typeof frame.id === 'number') { - this.rejectPending(frame.id, invalidAcpResponseEnvelope(record)) - } - return - } - if (typeof frame.id !== 'number') { - return - } - const pending = this.pending.get(frame.id) - if (!pending) { - return + settleAcpReply({ + record, + frame, + pending: this.pending, + rejectPending: (id, error) => this.rejectPending(id, error), + diagnose: (message) => this.diagnose(message) + }) + if (this.outputEnd.error && this.pending.size === 0) { + this.close(this.outputEnd.error) } - this.pending.delete(frame.id) - clearTimeout(pending.timer) - settleAcpResponse(frame, pending, (message) => this.diagnose(message)) } } diff --git a/src/main/acp/acp-peer-reply.ts b/src/main/acp/acp-peer-reply.ts new file mode 100644 index 000000000000..4366f8fd783b --- /dev/null +++ b/src/main/acp/acp-peer-reply.ts @@ -0,0 +1,42 @@ +import { z } from 'zod' +import type { AcpPendingResponse } from './acp-response-settlement' +import { + invalidAcpResponseEnvelope, + settleAcpResponse, + type AcpJsonRpcMessage +} from './acp-rpc-envelope' + +type RejectPending = (id: number, error: (method: string) => Error) => void + +/** A record no envelope reads still fails the call its id names. */ +export function rejectUnreadableAcpReply(record: unknown, rejectPending: RejectPending): void { + const response = z.object({ id: z.number(), method: z.undefined().optional() }).safeParse(record) + if (response.success) { + rejectPending(response.data.id, invalidAcpResponseEnvelope(record)) + } +} + +/** Settles the outstanding call an agent's reply answers; a malformed reply fails its call. */ +export function settleAcpReply(input: { + record: unknown + frame: AcpJsonRpcMessage + pending: Map + rejectPending: RejectPending + diagnose: (message: string) => void +}): void { + const { frame } = input + if ('result' in frame === 'error' in frame) { + input.diagnose('Ignored invalid ACP response') + if (typeof frame.id === 'number') { + input.rejectPending(frame.id, invalidAcpResponseEnvelope(input.record)) + } + return + } + const pending = typeof frame.id === 'number' ? input.pending.get(frame.id) : undefined + if (!pending || typeof frame.id !== 'number') { + return + } + input.pending.delete(frame.id) + clearTimeout(pending.timer) + settleAcpResponse(frame, pending, input.diagnose) +} diff --git a/src/main/acp/acp-scripted-connection.test-fixture.ts b/src/main/acp/acp-scripted-connection.test-fixture.ts index 05f15498a1fd..c121312c3348 100644 --- a/src/main/acp/acp-scripted-connection.test-fixture.ts +++ b/src/main/acp/acp-scripted-connection.test-fixture.ts @@ -5,7 +5,7 @@ import { type ProviderSpawnAbsence } from '../provider-process/provider-executable-missing' import type { AcpAgentConnectionOptions } from './acp-agent-connection' -import { AcpConnectionClosedError } from './acp-errors' +import { AcpAgentExitedError, AcpConnectionClosedError } from './acp-errors' import { AcpSessionRuntime } from './acp-session-runtime' import type { AcpStructuredConnection } from './acp-structured-connection' import { AcpScriptedAgent } from './acp-scripted-agent.test-support' @@ -80,7 +80,10 @@ export class FakeAcpChild extends AcpSessionRuntime implements AcpStructuredConn this.exitedWith = status // As the real connection: a spawn ENOENT marks the exit error with what it proved absent. const error = withProviderSpawnAbsence( - this.lostWith ?? new AcpConnectionClosedError(this.stderr || `${this.launch.command} exited`), + this.lostWith ?? + (this.closing + ? new AcpConnectionClosedError(this.stderr || `${this.launch.command} exited`) + : new AcpAgentExitedError(this.stderr || `${this.launch.command} exited`, this.stderr)), this.spawnAbsence ) super.close(error) diff --git a/src/main/acp/acp-session-reopen-failure.test.ts b/src/main/acp/acp-session-reopen-failure.test.ts new file mode 100644 index 000000000000..1a75b3ec7471 --- /dev/null +++ b/src/main/acp/acp-session-reopen-failure.test.ts @@ -0,0 +1,133 @@ +import { openAcpAdapterRig } from './acp-structured-adapter.test-support' +import { closeProviderTimelineRigs } from '../native-chat/agent-session-timeline/provider-timeline-assembler-test-support' +import { afterEach, expect, it, vi } from 'vitest' +import { PassThrough } from 'node:stream' +import { createFakePipedChild } from '../../shared/__fixtures__/fake-spawned-child' +import { PROVIDER_EXIT_OUTPUT_DRAIN_MS } from '../provider-process/provider-exit-output-drain' +import { createAcpAgentConnection } from './acp-agent-connection' +import { AcpAgentError, AcpRequestTimeoutError } from './acp-errors' +import { AcpScriptedAgent } from './acp-scripted-agent.test-support' +import { agentSessionFailureFact } from '../../shared/agent-session/failures/agent-session-failure' +import { agentSessionFailureWords } from '../../shared/agent-session/failures/agent-session-failure-words' +import { acpReopenTakeover } from './acp-session-reopen-failure' + +it.each(['ServerUnavailable', 'CatalogNotReady'])( + 'retains the saved session on a temporary %s load answer', + (errorName) => { + const error = new AcpAgentError(-32603, 'not ready', { errorName }) + const warn = vi.fn() + expect(() => + acpReopenTakeover( + error, + { + key: 'saved-key', + sessionId: 'saved-id', + mayBeUnsaved: () => false, + unannouncedLosses: () => [] + }, + { over: false, now: 1, warn } + ) + ).toThrow(error) + expect(warn).not.toHaveBeenCalled() + } +) + +afterEach(async () => { + vi.useRealTimers() + await closeProviderTimelineRigs() +}) + +it('propagates a temporary load envelope before takeover or a dead-child assertion can replace it', async () => { + const rig = await openAcpAdapterRig({ + launch: { + resume: { + key: 'saved-key', + sessionId: 'saved-id', + mayBeUnsaved: () => false, + unannouncedLosses: () => [] + } + }, + script: (agent) => + agent.on('session/load', (frame) => + agent.fail(frame, -32603, 'server unavailable', { errorName: 'ServerUnavailable' }) + ) + }) + await rig.acquire() + // The adapter answers for its own declared-temporary answer, which it keeps as the cause. + expect(rig.openVerdict()).toMatchObject({ retry: 'safe' }) + expect(rig.openFailure()).toBeInstanceOf(AcpAgentError) + expect(rig.openFailure()).toMatchObject({ data: { errorName: 'ServerUnavailable' } }) + expect(rig.sent('session/new')).toEqual([]) + expect(rig.sent('session/load')[0].params).toMatchObject({ sessionId: 'saved-id' }) + await rig.adapter.closeAll() +}) + +it.each([ + { agent: 'exited while its output drained', exits: true }, + { agent: 'is still running', exits: false } +])("keeps a load deadline's words when the agent $agent", async ({ exits }) => { + vi.useFakeTimers() + const spawned: ReturnType[] = [] + const agents: AcpScriptedAgent[] = [] + const rig = await openAcpAdapterRig({ + launch: { + resume: { + key: 'saved-key', + sessionId: 'saved-id', + mayBeUnsaved: () => false, + unannouncedLosses: () => [] + } + }, + deps: { + connect: (launch, options) => { + const agent = new AcpScriptedAgent() + agent.on('initialize', (frame) => + agent.reply(frame, { protocolVersion: 1, agentCapabilities: { loadSession: true } }) + ) + // The agent never answers the load, but would open a replacement session. + agent.on('session/load', () => {}) + agent.on('session/new', (frame) => agent.reply(frame, { sessionId: 'replacement-id' })) + const child = Object.assign(createFakePipedChild(), { + pid: 9_999_999, + stdout: agent.stdout, + stdin: agent.stdin, + stderr: new PassThrough(), + kill: () => true + }) + // A live agent exits when Orca closes its input. + agent.stdin.once('finish', () => { + if (!exits) { + child.emit('exit', 0, null) + } + }) + spawned.push(child) + agents.push(agent) + return createAcpAgentConnection(launch, options, () => child) + } + } + }) + try { + const acquired = rig.acquire() + await vi.advanceTimersByTimeAsync(299_500) + // An exiting agent leaves unanswered at 299.5 s, holding stdout open; the load's own deadline + // fires at 300 s, inside the output drain that the exit started. + if (exits) { + spawned[0].emit('exit', 1, null) + } + await vi.advanceTimersByTimeAsync(500 + PROVIDER_EXIT_OUTPUT_DRAIN_MS) + // No replacement session is opened, even in a live agent that would answer one. + expect(agents[0].frames.map((frame) => frame.method)).not.toContain('session/new') + await acquired + const timedOut = agentSessionFailureFact('startFailed', { + requestTimeout: { method: 'session/load', timeoutMs: 300_000 } + }) + expect(rig.openVerdict()).toMatchObject({ retry: 'never', fact: timedOut }) + expect(rig.openFailure()).toBeInstanceOf(AcpRequestTimeoutError) + expect(agentSessionFailureWords(rig.openVerdict()!.fact, { surface: 'rejection' })).toEqual( + agentSessionFailureWords(timedOut, { surface: 'rejection' }) + ) + expect(spawned).toHaveLength(1) + } finally { + await rig.adapter.closeAll() + } +}) diff --git a/src/main/acp/acp-session-reopen-failure.ts b/src/main/acp/acp-session-reopen-failure.ts index f24ae0036251..6e653694eacb 100644 --- a/src/main/acp/acp-session-reopen-failure.ts +++ b/src/main/acp/acp-session-reopen-failure.ts @@ -4,6 +4,9 @@ import type { AgentSessionProviderHandleLink } from '../../shared/agent-session/ import type { ProviderTimelineEvent } from '../native-chat/agent-session-timeline/provider-timeline-event' import { AcpAuthRequiredError, AcpRpcError } from './acp-errors' import type { AcpStructuredLaunch } from './acp-structured-launch-resolution' +import { providerRequestTimeoutOf } from '../provider-process/provider-request-timeout' + +import { temporaryAcpStartupAnswer } from './acp-startup-outcome' const ACP_RESOURCE_NOT_FOUND = -32002 export const ACP_REOPEN_FAILED = 'ACP agent could not reopen its saved session; starting a new one' @@ -13,8 +16,14 @@ export function acpReopenTakeover( resume: NonNullable, start: { over: boolean; now: number; warn: (fields: { scope: string; error: unknown }) => void } ): Pick { - // A replacement session cannot resolve sign-in failure or an abandoned start. - if (error instanceof AcpAuthRequiredError || start.over) { + // A replacement session cannot resolve sign-in failure, a missed deadline or an abandoned start; + // its own answer would replace the failure the user must see. + if ( + error instanceof AcpAuthRequiredError || + temporaryAcpStartupAnswer(error) || + providerRequestTimeoutOf(error) || + start.over + ) { throw error } if ( diff --git a/src/main/acp/acp-session-runtime.ts b/src/main/acp/acp-session-runtime.ts index 824b79b41b74..ce008176a246 100644 --- a/src/main/acp/acp-session-runtime.ts +++ b/src/main/acp/acp-session-runtime.ts @@ -3,7 +3,12 @@ import { z } from 'zod' import { AcpRpcError } from './acp-errors' import { AcpPromptRequest } from './acp-prompt-request' import { callAcpSession } from './acp-response-settlement' -import { AcpJsonRpcPeer, type AcpPeerOptions, type AcpRequestContext } from './acp-json-rpc-peer' +import { + AcpJsonRpcPeer, + type AcpPeerHandlers, + type AcpPeerOptions, + type AcpRequestContext +} from './acp-json-rpc-peer' import { answerAcpPermission, readAcpPermissionRequest, @@ -54,6 +59,7 @@ export type AcpSessionRuntimeOptions = { onExtensionNotification?: (method: string, params: unknown) => void onDiagnostic?: (message: string) => void onClose?: (error: Error) => void + onOutputEnd?: AcpPeerHandlers['onOutputEnd'] onOtherLine?: (line: string, cut?: boolean) => void /** Authentication answers, published synchronously as responses are accepted. */ onAuthAnswered?: (signedIn: boolean) => void @@ -86,6 +92,7 @@ export class AcpSessionRuntime { this.prompts.close(error) options.onClose?.(error) }, + ...(options.onOutputEnd ? { onOutputEnd: options.onOutputEnd } : {}), onOtherLine: options.onOtherLine }, options.peer diff --git a/src/main/acp/acp-startup-outcome.ts b/src/main/acp/acp-startup-outcome.ts new file mode 100644 index 000000000000..b4076070a2df --- /dev/null +++ b/src/main/acp/acp-startup-outcome.ts @@ -0,0 +1,49 @@ +import { + agentSessionAcquireFailure, + agentSessionStartFailure, + AgentSessionStartFailure, + type AgentSessionStartAnswer +} from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { + AcpAgentError, + AcpAgentExitedError, + AcpFrameTooLargeError, + AcpRpcError +} from './acp-errors' + +export function temporaryAcpStartupAnswer(error: unknown): boolean { + return ( + error instanceof AcpAgentError && + error.code === -32603 && + typeof error.data === 'object' && + error.data !== null && + 'errorName' in error.data && + (error.data.errorName === 'ServerUnavailable' || error.data.errorName === 'CatalogNotReady') + ) +} + +/** An agent's answer, sign-in included, is final unless it declared a temporary one; a frame + * Orca could not read is final too. Only the process's own drained exit is unanswered: a closed + * connection alone may be Orca refusing what the agent wrote. */ +export const acpStartupAnswer: AgentSessionStartAnswer = (cause) => { + if (cause instanceof AcpRpcError) { + return temporaryAcpStartupAnswer(cause) ? 'temporary' : 'final' + } + if (cause instanceof AcpFrameTooLargeError) { + return 'final' + } + return cause instanceof AcpAgentExitedError ? 'exited' : undefined +} + +export function acpStartFailure(error: unknown, exited = false): AgentSessionStartFailure { + return agentSessionStartFailure(error, acpStartupAnswer, exited) +} + +/** Orca closed the connection before the agent exited, refusing what it wrote: final. */ +export function acpRefusedStart(lost: Error): AgentSessionStartFailure { + return new AgentSessionStartFailure(lost, 'never') +} + +export function acpAcquireFailure(error: unknown): unknown { + return agentSessionAcquireFailure(error, acpStartupAnswer) +} diff --git a/src/main/acp/acp-structured-adapter.test-support.ts b/src/main/acp/acp-structured-adapter.test-support.ts index 2f4691585981..96d28fd26a30 100644 --- a/src/main/acp/acp-structured-adapter.test-support.ts +++ b/src/main/acp/acp-structured-adapter.test-support.ts @@ -19,6 +19,7 @@ import { type AcpScriptedAgent, tick, type FakeFrame } from './acp-scripted-agen import type { AcpStructuredLaunch } from './acp-structured-launch-resolution' import { AcpStructuredSessionAdapter } from './acp-structured-session-adapter' import type { AcpStructuredSessionAdapterDeps } from './acp-structured-session-adapter-deps' +import { AgentSessionStartFailure } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' export const GROK = acpLaunchSpecFor('grok')! const RIG_ENV = { PATH: '/usr/bin', ORCA_PANE_KEY: 'tab-1:pane-1', ORCA_AGENT_HOOK_PORT: '1234' } @@ -57,7 +58,10 @@ export type AcpAdapterRig = { lifecycle: StructuredAgentSessionLifecycleEvent[] readonly ended: Extract[] started(): StructuredAgentSessionProviderOpenResult + /** The error the adapter's verdict carries, as the agent or connection gave it. */ openFailure(): unknown + /** The adapter's verdict on the failed open. */ + openVerdict(): AgentSessionStartFailure | undefined settled: Parameters>[0][] acquire(options?: { fence?: number @@ -86,7 +90,7 @@ export async function openAcpAdapterRig( const spawned: string[] = [] const lifecycle: StructuredAgentSessionLifecycleEvent[] = [] const settled: AcpAdapterRig['settled'] = [] - let openFailure: unknown + let openVerdict: AgentSessionStartFailure | undefined let opened: StructuredAgentSessionProviderOpenResult | null = null const startListeners = new Set<() => void>() const spec = options.spec ?? GROK @@ -185,10 +189,11 @@ export async function openAcpAdapterRig( } return opened }, - openFailure: () => openFailure, + openFailure: () => openVerdict?.cause, + openVerdict: () => openVerdict, settled, acquire: async (acquireOptions = {}) => { - openFailure = undefined + openVerdict = undefined opened = null const acquisition = await adapter.acquire({ identity: { @@ -218,7 +223,10 @@ export async function openAcpAdapterRig( signal: acquireOptions.signal ?? new AbortController().signal }) } catch (error) { - openFailure = error + if (!(error instanceof AgentSessionStartFailure)) { + throw new Error('the ACP adapter must answer a failed open with its verdict') + } + openVerdict = error await adapter.closeSession(SESSION) } } diff --git a/src/main/acp/acp-structured-child-lifecycle.ts b/src/main/acp/acp-structured-child-lifecycle.ts index 2aaa3bf4854d..35a4359aa791 100644 --- a/src/main/acp/acp-structured-child-lifecycle.ts +++ b/src/main/acp/acp-structured-child-lifecycle.ts @@ -22,6 +22,9 @@ import { neverThrowingStructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' import { windDownAcpTurn } from './acp-structured-stop' +import { AcpAgentExitedError } from './acp-errors' +import { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { acpRefusedStart, acpStartFailure } from './acp-startup-outcome' /** An agent that never ran names its missing CLI or folder; any other end while starting is its exit. */ function acpStartupEndFailure(session: AcpStructuredChild): SubmissionRejectionFact { @@ -37,6 +40,8 @@ function acpStartupEndFailure(session: AcpStructuredChild): SubmissionRejectionF export class AcpStructuredChildren { readonly sessions = new Map() + /** Children that ended before they were ready answer for that ending, open included. */ + readonly endings = new AgentSessionStartEndings() readonly starts = new ProviderAcquisitionStarts() constructor(private readonly deps: AcpStructuredSessionAdapterDeps) {} capturePendingOutput: StructuredAgentSessionAdapter['capturePendingOutput'] = (input) => { @@ -184,6 +189,31 @@ export class AcpStructuredChildren { } } + /** Taken when the process exit is first seen, so a start settling meanwhile cannot miss it. A + * connection Orca closed first keeps that cause; only a bare exit is the agent's own. */ + recordExit( + sessionId: string, + generation: string, + connection: AcpStructuredConnection, + lost?: Error + ): void { + const detail = + connection.stderrTail() || `${this.deps.spec.agent} ACP agent ended while starting` + const exited = withProviderSpawnAbsence( + lost ?? new AcpAgentExitedError(detail, connection.stderrTail()), + connection.spawnAbsence + ) + this.endings.record( + sessionId, + generation, + lost ? acpRefusedStart(exited) : acpStartFailure(exited) + ) + } + + private recordEnding(session: AcpStructuredChild, error: Error): void { + this.endings.record(session.sessionId, session.acquisitionGeneration, acpStartFailure(error)) + } + /** Close the journal on protocol loss; notify the host after process exit is proven. */ connectionLost(session: AcpStructuredChild | null, error: Error): void { if ( @@ -193,6 +223,8 @@ export class AcpStructuredChildren { ) { return } + // Orca refusing what the agent wrote outranks the exit its cleanup then causes. + this.recordEnding(session, error) if (!isAcpReadySession(session)) { session.journalClosed = error.message || error.name return diff --git a/src/main/acp/acp-structured-connection.ts b/src/main/acp/acp-structured-connection.ts index a4a3ded7c6a1..630b4ca9a870 100644 --- a/src/main/acp/acp-structured-connection.ts +++ b/src/main/acp/acp-structured-connection.ts @@ -24,7 +24,10 @@ export type AcpStructuredConnection = Pick< | 'resumeReading' | 'close' | 'processTreeUnproven' -> & { readonly lastCloseResult?: AcpAgentConnection['lastCloseResult'] } +> & { + readonly lastCloseResult?: AcpAgentConnection['lastCloseResult'] + readonly lostBeforeExit?: AcpAgentConnection['lostBeforeExit'] +} export type ConnectAcpAgent = ( launch: ProviderProcessLaunch, diff --git a/src/main/acp/acp-structured-handshake.ts b/src/main/acp/acp-structured-handshake.ts index 99889464c465..3ba1b4f1dcfb 100644 --- a/src/main/acp/acp-structured-handshake.ts +++ b/src/main/acp/acp-structured-handshake.ts @@ -87,9 +87,9 @@ export async function initializeAcpStructuredSession( replayed = input.lanes.finishLoad() attaching.translator.finishLoad() } catch (error) { - assertStarting() takeover = acpReopenTakeover(error, resume, { - over: connection.closed || input.abandoned(), + // An agent already exited, its output still draining, has no session to take over in. + over: connection.closed || connection.exited || input.abandoned(), now: now(), warn: (fields) => deps.logger?.warn(ACP_REOPEN_FAILED, { ...fields, sessionId }) }) diff --git a/src/main/acp/acp-structured-publish-at-spawn.test.ts b/src/main/acp/acp-structured-publish-at-spawn.test.ts index da74d6cf7855..0d1d7b25457c 100644 --- a/src/main/acp/acp-structured-publish-at-spawn.test.ts +++ b/src/main/acp/acp-structured-publish-at-spawn.test.ts @@ -195,7 +195,11 @@ describe('ACP child published before its handshake', () => { acquisitionGeneration: acquired.acquisitionGeneration, signal: new AbortController().signal }) - ).rejects.toMatchObject({ reason: 'notSignedIn' }) + ).rejects.toMatchObject({ + retry: 'never', + fact: { kind: 'notSignedIn' }, + cause: { reason: 'notSignedIn' } + }) expect(rig.ended).toEqual([]) await rig.adapter.closeSession(SESSION) expect(rig.child().exited).toBe(true) diff --git a/src/main/acp/acp-structured-session-adapter-teardown.test.ts b/src/main/acp/acp-structured-session-adapter-teardown.test.ts index a453e5c27363..f3f215477e83 100644 --- a/src/main/acp/acp-structured-session-adapter-teardown.test.ts +++ b/src/main/acp/acp-structured-session-adapter-teardown.test.ts @@ -13,6 +13,7 @@ import { } from './acp-structured-adapter.test-support' import { framesOf, openAttachedHostRig, send } from './acp-structured-host.test-support' import { ACP_STOP_GRACE_MS } from './acp-structured-session-adapter-deps' +import { PROVIDER_EXIT_OUTPUT_DRAIN_MS } from '../provider-process/provider-exit-output-drain' afterEach(async () => { vi.useRealTimers() @@ -147,7 +148,10 @@ describe('ACP teardown with nothing to cancel', () => { return proves } child.agent.stdin.destroy() - await vi.waitFor(() => expect(child.closes).toBe(1)) + // The running prompt may still be answered, so the broken connection closes after the drain. + await vi.waitFor(() => expect(child.closes).toBe(1), { + timeout: PROVIDER_EXIT_OUTPUT_DRAIN_MS + 1_000 + }) proves = true await expect(rig.adapter.closeSession(SESSION)).resolves.toBe(true) expect(cancel).not.toHaveBeenCalled() diff --git a/src/main/acp/acp-structured-session-adapter.ts b/src/main/acp/acp-structured-session-adapter.ts index d9f8fd261ba0..3a3378cc19aa 100644 --- a/src/main/acp/acp-structured-session-adapter.ts +++ b/src/main/acp/acp-structured-session-adapter.ts @@ -1,11 +1,9 @@ import { randomUUID } from 'node:crypto' -import { withProviderDiagnostic } from '../../shared/agent-session/failures/agent-session-failure' import type { AgentSessionExecutionLocation } from '../../shared/agent-session/lifecycle/agent-session-record' import { AgentSessionAcquisitionExitProvenError, AgentSessionAcquisitionExitUnprovenError, AgentSessionAcquisitionRootExitObservedError, - AgentSessionAcquisitionRefusal, AgentSessionPreSpawnError, isAgentSessionPreSpawnError, type AgentSessionAcquisition, @@ -39,6 +37,8 @@ import { AcpStructuredChildren } from './acp-structured-child-lifecycle' import { AcpConversationCopies } from './acp-conversation-copies' import { AcpTextTasks } from './acp-text-task' import { supportsAcpStructuredLocation } from './acp-structured-location-support' +import { acpAcquireFailure, acpRefusedStart } from './acp-startup-outcome' +import { openAcpStart } from './acp-structured-start-open' import { admitStructuredAgentSessionInput } from '../native-chat/agent-session-wire/structured-agent-session-input-admission' export class AcpStructuredSessionAdapter implements SessionAdapter { @@ -77,6 +77,8 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { ) } return await this.start({ ...input, signal: attempt.signal }, attempt) + } catch (error) { + throw acpAcquireFailure(error) } finally { this.children.starts.end(attempt) gate?.leave() @@ -89,6 +91,7 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { ): Promise { const sessionId = input.identity.sessionId const generation = this.deps.mintGeneration?.() ?? randomUUID() + this.children.endings.admit(sessionId, generation) try { const { acquisition, session } = await acquireAcpStructuredSession({ acquire: input, @@ -96,6 +99,12 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { generation, abandoned: () => attempt.signal.aborted, track: (connection) => { + connection.onExit((_exit, requested, lost) => { + if (!requested) { + input.events?.observeRootExit?.() + this.children.recordExit(sessionId, generation, connection, lost) + } + }) this.children.starts.track(attempt, connection) this.deps.spec.launchGate?.retain(connection) }, @@ -115,20 +124,28 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { this.children.sessions.set(sessionId, session) return acquisition } catch (error) { + this.children.endings.retire(sessionId, generation) const { connection } = attempt if (connection && error instanceof AcpConnectionClosedError) { // Wait briefly for process exit so startup failures include final diagnostics. await waitForAcpExit(connection, this.graceMs(), attempt.signal) } - // Capture spontaneous exit before close can classify it as requested. - const exitedOnItsOwn = connection?.exited === true && !attempt.signal.aborted - if (connection && !(await connection.close().catch(() => false))) { + // Orca closing the connection first, such as on a sign-in line it cannot read, is the start's + // cause; the exit its cleanup then causes is not the agent ending unanswered. Capture + // spontaneous exit before close can classify it as requested. + const exitedOnItsOwn = + connection?.exited === true && !attempt.signal.aborted && !connection.lostBeforeExit + const cleaned = connection && (await connection.close().catch(() => false)) + // Read after cleanup, which may be what observes the refusal; it outranks generic errors. + const refused = connection?.lostBeforeExit + const cause = refused ? acpRefusedStart(refused) : error + if (connection && !cleaned) { // Retain failed cleanup for retry before another child can spawn. this.children.starts.retainFailed(sessionId, connection) - throw new AgentSessionAcquisitionExitUnprovenError(error) + throw new AgentSessionAcquisitionExitUnprovenError(cause) } if (connection?.lastCloseResult?.killDelivered && !connection.exited) { - throw new AgentSessionAcquisitionRootExitObservedError(error, 'kill-delivered') + throw new AgentSessionAcquisitionRootExitObservedError(cause, 'kill-delivered') } if (attempt.signal.aborted) { const closed = new Error( @@ -138,8 +155,8 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { throw connection ? closed : new AgentSessionPreSpawnError(closed) } // Once the exit is proven, the start's error names a CLI or folder the spawn found missing. - if (error instanceof Error && connection?.exited) { - withProviderSpawnAbsence(error, connection.spawnAbsence) + if (cause instanceof Error && connection?.exited) { + withProviderSpawnAbsence(cause, connection.spawnAbsence) } if (connection && exitedOnItsOwn && !isAgentSessionPreSpawnError(error)) { throw new AgentSessionAcquisitionExitProvenError( @@ -148,34 +165,17 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { ) ) } - throw error + throw cause } } - open: NonNullable = async (input) => { - const child = this.children.sessions.get(input.sessionId) - if ( - !child || - child.fence !== input.fence || - child.acquisitionGeneration !== input.acquisitionGeneration || - child.ended - ) { - throw new Error('ACP child changed before it opened') - } - try { - return await child.open(input.signal) - } catch (error) { - const failure = childState.acpStartupFailure(child, error) - if (failure) { - const refusal = new AgentSessionAcquisitionRefusal( - `${acpAgentName(child.spec.agent)} requires sign-in`, - 'notSignedIn' - ) - throw failure.detail ? withProviderDiagnostic(refusal, failure.detail) : refusal - } - throw error - } - } + open: NonNullable = (input) => openAcpStart(this.children, input) + + startupEnding: NonNullable = (sessionId, generation) => + this.children.endings.get(sessionId, generation) + + retireStartup: NonNullable = (sessionId, generation) => + this.children.endings.retire(sessionId, generation) async dispatch(input: Parameters[0]): Promise { const lost = this.children.sessions.get(input.sessionId) @@ -311,15 +311,20 @@ export class AcpStructuredSessionAdapter implements SessionAdapter { backgroundTaskStops = (sessionId: string) => acpChildStopCapabilities(this.children.sessions.get(sessionId)) - closeSession = (sessionId: string): Promise => this.children.close(sessionId) - disposeSession = (sessionId: string): Promise => this.children.close(sessionId) - releaseAcquisition = (input: { sessionId: string }) => this.children.close(input.sessionId) + // A released or closed session's start has no reader left for its ending. + closeSession = (sessionId: string): Promise => { + this.children.endings.retire(sessionId) + return this.children.close(sessionId) + } + disposeSession = this.closeSession + releaseAcquisition = (input: { sessionId: string }) => this.closeSession(input.sessionId) hasOwnedAcquisition = (sessionId: string): boolean => this.children.sessions.has(sessionId) || this.children.starts.failedSessionIds().includes(sessionId) forceCloseSession = (sessionId: string): Promise => this.children.close(sessionId, false) async closeAll(): Promise { + this.children.endings.clear() await this.children.closeAll([ this.atRestCommands?.closeAll?.(), this.copies.closeAll(), diff --git a/src/main/acp/acp-structured-start-open.ts b/src/main/acp/acp-structured-start-open.ts new file mode 100644 index 000000000000..cb9034eed3b8 --- /dev/null +++ b/src/main/acp/acp-structured-start-open.ts @@ -0,0 +1,45 @@ +import { withProviderDiagnostic } from '../../shared/agent-session/failures/agent-session-failure' +import { + AgentSessionAcquisitionRefusal, + type StructuredAgentSessionAdapter +} from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { StructuredAgentSessionProviderOpenInput } from '../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' +import { acpAgentName } from './acp-structured-agent-definitions' +import { acpStartupFailure } from './acp-structured-child' +import type { AcpStructuredChildren } from './acp-structured-child-lifecycle' +import { acpStartFailure } from './acp-startup-outcome' + +/** A start's open answers with the agent's verdict; a child that already ended answers with how + * it ended, and an answer its own exit followed is still worded as that exit. */ +export async function openAcpStart( + children: AcpStructuredChildren, + input: StructuredAgentSessionProviderOpenInput +): ReturnType> { + const child = children.sessions.get(input.sessionId) + try { + if ( + !child || + child.fence !== input.fence || + child.acquisitionGeneration !== input.acquisitionGeneration || + child.ended + ) { + throw ( + children.endings.get(input.sessionId, input.acquisitionGeneration) ?? + new Error('ACP child changed before it opened') + ) + } + return await child.open(input.signal).catch((error: unknown) => { + const failure = acpStartupFailure(child, error) + if (!failure) { + throw error + } + const refusal = new AgentSessionAcquisitionRefusal( + `${acpAgentName(child.spec.agent)} requires sign-in`, + 'notSignedIn' + ) + throw failure.detail ? withProviderDiagnostic(refusal, failure.detail) : refusal + }) + } catch (error) { + throw acpStartFailure(error, child?.connection.exited === true && !child.closeRequested) + } +} diff --git a/src/main/claude/claude-agent-sdk-control-requests.ts b/src/main/claude/claude-agent-sdk-control-requests.ts index 066467060982..980093a4e9c3 100644 --- a/src/main/claude/claude-agent-sdk-control-requests.ts +++ b/src/main/claude/claude-agent-sdk-control-requests.ts @@ -7,9 +7,10 @@ import type { export class ClaudeControlRequestError extends Error { constructor( readonly subtype: string, - message: string + message: string, + options?: ErrorOptions ) { - super(message) + super(message, options) this.name = 'ClaudeControlRequestError' } } @@ -27,6 +28,10 @@ export class ClaudeControlRequestTimeoutError extends Error { // Exact SDK error used to distinguish query closure from CLI rejection. const QUERY_CLOSED_MESSAGE = 'Query closed before response received' +export function isClaudeQueryClosed(error: unknown): boolean { + return error instanceof Error && error.message === QUERY_CLOSED_MESSAGE +} + // The SDK implements getSettings but omits it from Query's declaration. type ClaudeQuerySettingsReader = { getSettings?: () => Promise } @@ -72,7 +77,7 @@ export function runClaudeControl( if (error instanceof ClaudeControlRequestError || message === QUERY_CLOSED_MESSAGE) { throw error } - throw new ClaudeControlRequestError(subtype, message) + throw new ClaudeControlRequestError(subtype, message, { cause: error }) }) if (timeoutMs === null) { return request diff --git a/src/main/claude/claude-startup-answer-observation.ts b/src/main/claude/claude-startup-answer-observation.ts new file mode 100644 index 000000000000..ec460aa09eb6 --- /dev/null +++ b/src/main/claude/claude-startup-answer-observation.ts @@ -0,0 +1,55 @@ +import { StringDecoder } from 'node:string_decoder' +import { createIncrementalNdjsonFramer } from '../../shared/main-process-ndjson-framer' + +// Unescaped quotes occur only as JSON structure, so other records skip this second parse. +const FAILED_RESULT_TOKENS = [/"type"\s*:\s*"result"/, /"is_error"\s*:\s*true/] + +function mayBeFailedResult(text: string): boolean { + return FAILED_RESULT_TOKENS.every((token) => token.test(text)) +} + +/** Observe before SDK cleanup can wait on transcript flushing; feed is the connection's one stdout tap. */ +export function claudeStartupAnswerObservation(next?: (chunk: Buffer | string) => void) { + let failure: Error | undefined + let done = false + const decoder = new StringDecoder('utf8') + const stop = (): void => { + done = true + framer.reset() + } + const observe = (record: unknown): void => { + if ( + typeof record === 'object' && + record !== null && + 'type' in record && + record.type === 'result' && + 'is_error' in record && + record.is_error === true && + !('parent_tool_use_id' in record && record.parent_tool_use_id) + ) { + failure ??= Object.assign(new Error('claude reported a failed result during startup'), { + errorClass: 'error_result' + }) + stop() + } + } + // The shared line budget bounds retention; a rejected prefix cannot prove a root record. + // A capture ends parsing mid-chunk; the rest of that chunk only reaches the next handler. + const framer = createIncrementalNdjsonFramer(observe, () => {}, { + acceptLine: (line) => !done && mayBeFailedResult(line) + }) + return { + failure: () => failure, + finish: stop, + // Observes until capture, send or close; every chunk still reaches the next handler. + feed: (chunk: Buffer | string): void => { + if (!done) { + framer.feed(typeof chunk === 'string' ? chunk : decoder.write(chunk)) + if (done) { + framer.reset() + } + } + next?.(chunk) + } + } +} diff --git a/src/main/claude/claude-startup-failure.ts b/src/main/claude/claude-startup-failure.ts new file mode 100644 index 000000000000..f7eeec57316b --- /dev/null +++ b/src/main/claude/claude-startup-failure.ts @@ -0,0 +1,44 @@ +import { findInErrorChain } from '../provider-process/provider-error-chain' +import { ClaudeControlRequestError } from './claude-agent-sdk-control-requests' +import { + agentSessionAcquireFailure, + agentSessionStartFailure, + providerResourcePressure, + type AgentSessionStartAnswer, + type AgentSessionStartFailure +} from '../native-chat/agent-session-wire/structured-agent-session-start-failure' + +/** SDK runtime tags are not in its public Error declaration. */ +export function claudeStartupErrorClass(error: Error): string | undefined { + return 'errorClass' in error && typeof error.errorClass === 'string' + ? error.errorClass + : undefined +} + +/** The SDK's tag reports whether Claude's process ended unanswered; every other tag is its answer. + * A launch the host could not run for want of resources is left to the shared rule. */ +export const claudeStartupAnswer: AgentSessionStartAnswer = (error) => { + const tag = + claudeStartupErrorClass(error) ?? + (error instanceof ClaudeControlRequestError + ? (findInErrorChain(error.cause, (cause) => claudeStartupErrorClass(cause)) ?? 'answered') + : undefined) + if (tag === undefined) { + return undefined + } + if (tag === 'process_exited_nonzero' || tag === 'process_killed_by_signal') { + return 'reported' + } + if (tag === 'executable_launch_failed' || tag === 'spawn_failed' || tag === 'process_error') { + return providerResourcePressure(error) ? undefined : 'final' + } + return 'final' +} + +export function claudeStartFailure(error: unknown): AgentSessionStartFailure { + return agentSessionStartFailure(error, claudeStartupAnswer) +} + +export function claudeAcquireFailure(error: unknown): unknown { + return agentSessionAcquireFailure(error, claudeStartupAnswer) +} diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index ad18841ffbd4..7d9ab1adef06 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -1,7 +1,7 @@ import { execFileSync } from 'node:child_process' import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { dirname, join } from 'node:path' import { providerDiagnosticOf } from '../../shared/agent-session/failures/agent-session-failure' import { afterEach, describe, expect, it, vi } from 'vitest' import { spawnProcess } from '@orca/process-host' @@ -24,6 +24,17 @@ import { import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { claudeProviderHandle } from '../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' import { providerExecutableMissing } from '../provider-process/provider-executable-missing' +import { findInErrorChain } from '../provider-process/provider-error-chain' +import { providerExitObserved } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { + adapterAtPublishFor, + fakeClaude, + identityFor +} from './structured/claude-structured-session-test-support' +import { claudeStartFailure } from './claude-startup-failure' + +const retryableStructuredAgentSessionStart = (error: unknown): boolean => + claudeStartFailure(error).retry === 'safe' // Exercise the real SDK against a scripted child CLI. const FAKE_CLI = join(__dirname, '__fixtures__', 'claude-agent-sdk-scripted-cli.mjs') @@ -502,6 +513,79 @@ describe('Claude stream-json connection', () => { } }) + it('lets a final initialize answer after a physical exit own the start despite fault-first callbacks', async () => { + const go = join(mkdtempSync(join(tmpdir(), 'claude-sdk-exit-answer-')), 'go') + scratchDirs.push(dirname(go)) + const scenario = scriptScenario([ + { awaitFile: { path: go, timeoutMs: 15_000 } }, + { delayMs: 20 }, + { exit: 1 } + ]) + // Real SDK and child; only initialize's answer is held so it can arrive after the exit. + const { query: sdkQuery } = await import('@anthropic-ai/claude-agent-sdk') + const answer = Promise.withResolvers() + const queryImpl: typeof query = (args) => + Object.assign(sdkQuery(args), { initializationResult: () => answer.promise }) + const reports: string[] = [] + const exited = Promise.withResolvers() + const claude = fakeClaude() + const adapter = adapterAtPublishFor({ + ...claude, + openConnection: (_launch, handlers = {}) => + open( + launchFor(scenario), + { + ...handlers, + onFault: (error) => { + reports.push(providerExitObserved(error) ? 'exit-fault' : 'fault') + handlers.onFault?.(error) + }, + onExit: (error, exit) => { + reports.push('exit') + handlers.onExit?.(error, exit) + exited.resolve() + } + }, + queryImpl + ) + }) + try { + const acquired = await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 's' }) + let settled = false + const opened = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .then( + () => null, + (error: unknown) => error + ) + .finally(() => { + settled = true + }) + writeFileSync(go, '') + await exited.promise + // A fault-first start would already have settled; one macrotask drains its rejection chain. + await new Promise((resolve) => setImmediate(resolve)) + + expect(reports).toEqual(['exit-fault', 'exit']) + expect(settled).toBe(false) + answer.reject( + Object.assign(new Error('initialize failed'), { errorClass: 'control_request_failed' }) + ) + const failure = await opened + expect( + findInErrorChain(failure, (inner) => ('errorClass' in inner ? inner.errorClass : undefined)) + ).toBe('control_request_failed') + expect(retryableStructuredAgentSessionStart(failure)).toBe(false) + } finally { + await adapter.closeAll() + } + }) + it.runIf(process.platform !== 'win32')( 'proves a natural SDK exit and cleans up its descendant before recovery', async () => { diff --git a/src/main/claude/claude-stream-json-connection.ts b/src/main/claude/claude-stream-json-connection.ts index a04b7ecd2a69..2d2c395d91a4 100644 --- a/src/main/claude/claude-stream-json-connection.ts +++ b/src/main/claude/claude-stream-json-connection.ts @@ -1,3 +1,4 @@ +import { claudeStartupAnswerObservation } from './claude-startup-answer-observation' import type { PipedProcessSpawner } from '@orca/process-host/process-spec' import type * as ClaudeAgentSdk from '@anthropic-ai/claude-agent-sdk' import type { CanUseTool, OnUserDialog, SDKUserMessage } from '@anthropic-ai/claude-agent-sdk' @@ -22,6 +23,7 @@ import { import type { ClaudeStructuredSdkOptions } from './structured/launch/claude-structured-launch-resolution' import { withProviderSpawnAbsence } from '../provider-process/provider-executable-missing' import { claudeStreamJsonExitError } from './claude-stream-json-exit-error' +import { withObservedProviderExit } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' export { ClaudeControlRequestError } @@ -67,7 +69,7 @@ export type ClaudeStreamJsonConnectionHandlers = { canUseTool?: CanUseTool /** `request_user_dialog` control; the CLI only emits kinds declared in `supportedDialogKinds`. */ onUserDialog?: OnUserDialog - /** A transport/process fault that is not itself first-hand root exit proof. */ + /** A transport/process fault; a physical root exit arrives marked observed and is followed by onExit. */ onFault?: (error: Error) => void /** Report observed root exit once; expected stays true even after an unproven close attempt. */ onExit?: (error: Error, exit?: { expected: boolean }) => void @@ -87,6 +89,8 @@ export type ClaudeStreamJsonConnection = ClaudeControlSurface & { /** What the ladder has observed so far; read after a `close()` that returned false. */ readonly exitVerdict: ClaudeChildExitVerdict readonly executableMissing?: boolean + exited?: Promise + startupFailure?: () => Error | undefined pauseReading?: () => void resumeReading?: () => void send: (message: Record, beforeDispatch?: () => Promise) => Promise @@ -101,12 +105,8 @@ export async function openClaudeStreamJsonConnection( queryImpl?: typeof ClaudeAgentSdk.query ): Promise { const { query } = await loadClaudeAgentSdk() - const spawner = createClaudeCodeProcessSpawn( - spawnImpl, - process.platform, - launch, - handlers.onStdout - ) + const startup = claudeStartupAnswerObservation(handlers.onStdout) + const spawner = createClaudeCodeProcessSpawn(spawnImpl, process.platform, launch, startup.feed) // Always supply scrubbed env to prevent ambient auth leaks; only launch.env may pin CLAUDE_CONFIG_DIR. const env = await withoutRejectedClaudeApiKey( withoutKeys( @@ -135,6 +135,7 @@ export async function openClaudeStreamJsonConnection( if (!child || !managed) { throw new Error('the claude agent SDK returned without spawning a child') } + const exited = Promise.withResolvers() let closing = false let terminalError: Error | null = null let faultReported = false @@ -185,6 +186,10 @@ export async function openClaudeStreamJsonConnection( const error = claudeStreamJsonExitError(managed.stderrTail(), managed.exitStatus, cause) terminalError = withProviderSpawnAbsence(error, managed.spawnAbsence) } + // One physical exit reaches both callbacks; mark it first so onFault cannot treat it as live-root. + if (managed.rootExitObserved && !closing) { + withObservedProviderExit(terminalError) + } inbox.fail(terminalError) if (!closing && !faultReported) { faultReported = true @@ -193,6 +198,7 @@ export async function openClaudeStreamJsonConnection( if (managed.rootExitObserved && !exitReported) { exitReported = true handlers.onExit?.(terminalError, { expected: closing }) + exited.resolve(terminalError) } } @@ -259,6 +265,7 @@ export async function openClaudeStreamJsonConnection( ) ) } + startup.finish() // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: dispatch constructs the SDK user envelope after mapping every content block. return inbox.push(message as unknown as SDKUserMessage, beforeDispatch) } @@ -292,13 +299,17 @@ export async function openClaudeStreamJsonConnection( pid: spawner.pid }) } + startup.finish() return true })() return closePromise } + const controls = createClaudeControlSurface(session) return { - ...createClaudeControlSurface(session), + ...controls, + startupFailure: startup.failure, + exited: exited.promise, get pid() { return spawner.pid }, diff --git a/src/main/claude/claude-stream-json-exit-error.test.ts b/src/main/claude/claude-stream-json-exit-error.test.ts index 0e36717ba7ff..a7c08e707bd1 100644 --- a/src/main/claude/claude-stream-json-exit-error.test.ts +++ b/src/main/claude/claude-stream-json-exit-error.test.ts @@ -5,7 +5,10 @@ import { claudeStreamJsonExitError } from './claude-stream-json-exit-error' import { providerStartupFailureFact } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' -import { retryableStructuredAgentSessionStart } from '../native-chat/agent-session-wire/structured-agent-session-start-retry' +import { claudeStartFailure } from './claude-startup-failure' + +const retryableStructuredAgentSessionStart = (error: unknown): boolean => + claudeStartFailure(error).retry === 'safe' const frame = JSON.parse( readFileSync(new URL('./__fixtures__/claude-missing-conversation.jsonl', import.meta.url), 'utf8') @@ -24,7 +27,7 @@ it('treats the recorded missing-conversation exit as an explained final refusal' expect(retryableStructuredAgentSessionStart(error)).toBe(false) }) -it('reads the same refusal from the recorded result, and leaves other startup results retryable', () => { +it('reads the same refusal from the recorded result, and makes other startup results final', () => { const refusal = claudeStartupResultError(frame) expect(providerStartupFailureFact(refusal)).toEqual({ kind: 'savedConversationMissing', @@ -36,5 +39,5 @@ it('reads the same refusal from the recorded result, and leaves other startup re kind: 'startFailed', detail: { text: 'managed settings failed', audience: 'log' } }) - expect(retryableStructuredAgentSessionStart(generic)).toBe(true) + expect(retryableStructuredAgentSessionStart(generic)).toBe(false) }) diff --git a/src/main/claude/fixtures/startup-sdk-process-exit.json b/src/main/claude/fixtures/startup-sdk-process-exit.json new file mode 100644 index 000000000000..b8329e128e11 --- /dev/null +++ b/src/main/claude/fixtures/startup-sdk-process-exit.json @@ -0,0 +1,22 @@ +{ + "sdk": "0.3.284", + "probe": "actual CLI invalid startup flag before initialize", + "records": [ + { + "initializationError": { + "name": "Error", + "message": "Claude Code process exited with code 1", + "errorClass": "process_exited_nonzero", + "exitCode": 1 + } + }, + { + "iteratorError": { + "name": "Error", + "message": "Claude Code process exited with code 1", + "errorClass": "process_exited_nonzero", + "exitCode": 1 + } + } + ] +} diff --git a/src/main/claude/structured/claude-structured-resume-point.ts b/src/main/claude/structured/claude-structured-resume-point.ts index 3fdb7d5967ae..1d6f01857dee 100644 --- a/src/main/claude/structured/claude-structured-resume-point.ts +++ b/src/main/claude/structured/claude-structured-resume-point.ts @@ -92,6 +92,20 @@ function persistClaudeTurnResumePoint( session.resumePointWrite = write } +/** A dispatch or a turn end since launch; without either, acquisition's saved cursor is still current. */ +export function claudeSessionHasTurnWork( + session: Pick< + ClaudeSession, + 'dispatchSequence' | 'resumePointWrite' | 'turnEndLeafUuid' | 'initialization' + > +): boolean { + return ( + session.dispatchSequence !== 0 || + session.resumePointWrite !== undefined || + session.turnEndLeafUuid !== session.initialization.history.leafUuid + ) +} + /** Drain turn-end writes before close or exit persists the final cursor. */ export async function settledClaudeTurnEndLeaf(session: ClaudeSession): Promise { await session.resumePointWrite?.settled diff --git a/src/main/claude/structured/claude-structured-session-acquisition.ts b/src/main/claude/structured/claude-structured-session-acquisition.ts index b42e1cd326e4..ec77bf36e4c8 100644 --- a/src/main/claude/structured/claude-structured-session-acquisition.ts +++ b/src/main/claude/structured/claude-structured-session-acquisition.ts @@ -24,7 +24,7 @@ import { } from './claude-structured-session-publication' import type * as sessionState from './claude-structured-session-state' import { resolveClaudeAcquisitionError } from './claude-structured-session-close' -import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { claudeStartupError, claudeStartupExit } from './claude-structured-session-startup-exit' import { readClaudeTranscriptEntryUuid } from '../claude-transcript-entry-uuid' import { claudeTurnResumePointWriter } from './claude-structured-resume-point' import { withAgentSessionCreatePhase } from '../../observability/agent-session-instrumentation' @@ -34,7 +34,7 @@ import { createClaudeSessionJournalTranslator } from './journal/claude-structured-session-journal-control' import { isRootClaudeFrame } from '../claude-turn-opening' -import { createClaudeStartupOutcome } from './claude-structured-startup-outcome' +import { createClaudeStartupOutcome, endsClaudeStartup } from './claude-structured-startup-outcome' import { claudeStartupResultError } from '../claude-stream-json-exit-error' export async function acquireClaudeSession({ @@ -62,9 +62,9 @@ export async function acquireClaudeSession({ const initProof = createClaudeInitProof() // Owned by this acquisition alone, so a late frame cannot fail a replacement's open. const startup = createClaudeStartupOutcome() - const childFailed = (error: Error): void => { + const childFailed = (error: Error, ended?: Promise): void => { childEnded ??= error - startup.fail(error) + void (ended ? ended.then(startup.fail) : endsClaudeStartup(error) && startup.fail(error)) initProof.reject(error) } const translator = createClaudeSessionJournalTranslator( @@ -222,8 +222,7 @@ export async function acquireClaudeSession({ callbacks.finishClose(sessionId, attempt) return } - // Mark in place because the fault report may retain this error object. - childFailed(withObservedProviderExit(error)) + childFailed(...claudeStartupExit(error, attempt.connection, input.events)) callbacks.handleExit(sessionId, attempt, error) }) } @@ -284,7 +283,7 @@ export async function acquireClaudeSession({ }) acquisitions.deleteIfCurrent(sessionId, attempt) await withAgentSessionCreatePhase('publish', input.recordPhase, async () => { - sessions.set(sessionId, session) + callbacks.publish(session) attempt.published = true for (const event of attempt.buffered.splice(0)) { event() @@ -300,7 +299,7 @@ export async function acquireClaudeSession({ } catch (error) { unbindReadingControl?.() throw await resolveClaudeAcquisitionError({ - error, + error: await claudeStartupError(error, attempt.connection), sessionId, sessions, attempt, diff --git a/src/main/claude/structured/claude-structured-session-adapter.ts b/src/main/claude/structured/claude-structured-session-adapter.ts index 392f4e6f5d9e..dd9db92aa341 100644 --- a/src/main/claude/structured/claude-structured-session-adapter.ts +++ b/src/main/claude/structured/claude-structured-session-adapter.ts @@ -1,6 +1,8 @@ -import { waitForPromiseWithSignal } from '../../../shared/abort-signal-reason' import { dispatchClaudeCommand } from './claude-structured-command-dispatch' import { AgentSessionAcquisitionRootExitObservedError } from '../../native-chat/agent-session-wire/structured-agent-session-acquisition-errors' +import { claudeAcquireFailure } from '../claude-startup-failure' +import { claudeStartPublication, openClaudeStart } from './claude-structured-start-ending' +import { AgentSessionStartEndings } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' import type { StructuredAgentSessionAcquireInput, StructuredAgentSessionAdapter @@ -43,7 +45,6 @@ import { } from './journal/claude-structured-prompt-ownership' import { claudePromptCancelRoute } from './journal/claude-structured-prompt-replies' import { claudeForkForRewind } from '../claude-rewind-fork' -import { openClaudeStructuredSession } from './claude-structured-session-open' import { ClaudeConversationNaming } from '../claude-conversation-naming' export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAdapter { @@ -51,6 +52,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda private readonly acquisitions = new ClaudeAcquisitionRegistry() private readonly exits = new Map() private readonly settledExitErrors = new Map() + private readonly endings = new AgentSessionStartEndings() private readonly exitLifecycle: ClaudeExitLifecycle private readonly naming: ClaudeConversationNaming | undefined @@ -62,6 +64,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda sessions: this.sessions, exits: this.exits, settledExitErrors: this.settledExitErrors, + endings: this.endings, deps, emit: (session, event) => this.emit(session, event) } @@ -71,8 +74,10 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda acquire = (input: StructuredAgentSessionAcquireInput) => { this.settledExitErrors.delete(input.identity.sessionId) + this.endings.retire(input.identity.sessionId) const record = this.deps.namingRecord?.(input.identity.sessionId) const naming = this.naming?.acquire(input, record) + const start = claudeStartPublication(this.sessions, this.endings, input.identity.sessionId) return acquireClaudeSession({ input, deps: this.deps, @@ -81,6 +86,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda exits: this.exits, ...(naming ? { naming } : {}), callbacks: { + publish: start.publish, deliver: (attempt, sessionId, event) => this.deliver(attempt, sessionId, event), emit: (session, _events, event) => this.emit(session, event), handleExit: (sessionId, attempt, error) => @@ -97,8 +103,9 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda settleClaudeUnexpectedExit(this.exitLifecycle, sessionId, exit) } }).catch((error: unknown) => { + start.failed() naming?.failed() - throw error + throw claudeAcquireFailure(error) }) } @@ -219,34 +226,26 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda input, this.deps.requestTimeoutMs ) - open: NonNullable = async (request) => { - try { - return await openClaudeStructuredSession({ - request, - session: this.session(request.sessionId), - sessions: this.sessions, - deps: this.deps, - emit: (session, event) => this.emit(session, event) - }) - } catch (error) { - const exit = this.exits.get(request.sessionId) - // Publish an already-observed exit before the host treats this rejection as its own fault. - if ( - exit?.session.acquisitionGeneration === request.acquisitionGeneration && - exit.publication - ) { - await waitForPromiseWithSignal(exit.publication, request.signal) - } - throw error - } - } + open: NonNullable = (request) => + openClaudeStart({ + request, + sessions: this.sessions, + exits: this.exits, + endings: this.endings, + deps: this.deps, + emit: (session, event) => this.emit(session, event) + }) readOptions = (input: { sessionId: string; fence: number }) => readClaudeStructuredSessionOptions(this.session(input.sessionId)) - releaseAcquisition = (input: { sessionId: string }): Promise => - this.afterClose(input.sessionId, () => this.releaseProviderSession(input.sessionId)) + releaseAcquisition = (input: { sessionId: string }): Promise => { + this.endings.retire(input.sessionId) + return this.afterClose(input.sessionId, () => this.releaseProviderSession(input.sessionId)) + } acknowledgeSessionRelease = (sessionId: string): void => { this.settledExitErrors.delete(sessionId) } + startupEnding = (sessionId: string, generation: string) => this.endings.get(sessionId, generation) + retireStartup = (id: string, generation: string) => this.endings.retire(id, generation) hasOwnedAcquisition = (sessionId: string): boolean => this.acquisitions.get(sessionId) !== undefined || this.sessions.has(sessionId) || @@ -296,14 +295,16 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } }) - closeAll = (): Promise => - closeAllClaudeSessions({ + closeAll = (): Promise => { + this.endings.clear() + return closeAllClaudeSessions({ sessions: this.sessions, acquisitions: this.acquisitions, exits: this.exits, closeSession: this.closeSession, closeExit: (sessionId) => this.releaseAcquisition({ sessionId }) }) + } private session(sessionId: string): ClaudeSession { const session = this.sessions.get(sessionId) diff --git a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts index 5a273542b870..77238390ba2e 100644 --- a/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/structured/claude-structured-session-exit-lifecycle.ts @@ -1,3 +1,4 @@ +import { withTimeout } from '../../../shared/promise-timeout-fallback' import { agentSessionFailureFact, providerDiagnosticOf @@ -8,8 +9,13 @@ import { startupAcquisitionRefusal } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' import { providerSpawnAbsent } from '../../provider-process/provider-executable-missing' -import { settledClaudeTurnEndLeaf } from './claude-structured-resume-point' +import { + claudeSessionHasTurnWork, + settledClaudeTurnEndLeaf +} from './claude-structured-resume-point' import { claudeHeldSends } from '../claude-command-lifecycle' +import type { AgentSessionStartEndings } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { claudeExitEnding } from './claude-structured-start-ending' import { claudeCloseReport, settleClaudeExitedSession } from './claude-structured-session-close' import { AgentSessionAcquisitionExitUnprovenError, @@ -28,6 +34,8 @@ export type ClaudeExitLifecycle = { exits: Map /** Preserve diagnostics for sends admitted before the host learns of the exit. */ settledExitErrors: Map + /** Taken when the exit is first seen, so a start settling meanwhile cannot miss it. */ + endings: AgentSessionStartEndings deps: Pick emit: (session: ClaudeSession, event: ClaudeStructuredSessionEvent) => void } @@ -44,9 +52,14 @@ export function observeClaudeSessionExit( } const observedAt = lifecycle.deps.now?.() ?? Date.now() session.events?.observeExit?.(claudeHeldSends(session), observedAt) + lifecycle.endings.record(sessionId, session.acquisitionGeneration, claudeExitEnding(error)) lifecycle.sessions.delete(sessionId) // Run tree cleanup before recovery; an exit callback proves only root exit. - const closePromise = session.connection.close().catch(() => false) + const cleanup = session.connection.close().catch(() => false) + const closePromise = + claudeStartDied(session) && providerExitObserved(error) + ? withTimeout(cleanup, 2_000, false) + : cleanup const exit: ClaudeSessionExit = { connection: session.connection, session, @@ -69,6 +82,12 @@ export function observeClaudeSessionExit( .catch(() => undefined) } +/** Exited before the host-awaited open returned and before any turn; turns can settle without an Orca dispatch. */ +function claudeStartDied(session: ClaudeSession): boolean { + // Initialize can answer while a resume open still waits on transcript catch-up. + return !session.initialization.startup.opened() && !claudeSessionHasTurnWork(session) +} + export function settleClaudeUnexpectedExit( lifecycle: ClaudeExitLifecycle, sessionId: string, @@ -81,10 +100,12 @@ export function settleClaudeUnexpectedExit( settleClaudeExitedSession(exit.session) return } - // Save the completed-turn cursor before recovery permits reacquisition. - await persistClaudeSessionHandle(sessionId, exit.session, deps).catch((error: unknown) => { - console.warn('[claude-resume-point] exit cursor was not persisted:', { sessionId, error }) - }) + // A dead start's host open is still waiting on this exit, and nothing advanced acquisition's cursor. + if (!claudeStartDied(exit.session)) { + await persistClaudeSessionHandle(sessionId, exit.session, deps).catch((error: unknown) => { + console.warn('[claude-resume-point] exit cursor was not persisted:', { sessionId, error }) + }) + } if (exits.get(sessionId) !== exit) { settleClaudeExitedSession(exit.session) return @@ -92,6 +113,9 @@ export function settleClaudeUnexpectedExit( exits.delete(sessionId) const report = claudeCloseReport(exit.connection, true, exit.error) lifecycle.settledExitErrors.set(sessionId, report ?? exit.error) + const startupFailure = claudeSessionHasTurnWork(exit.session) + ? undefined + : exit.connection.startupFailure?.() const ended: ClaudeStructuredSessionEvent = { type: 'ended', sessionId, @@ -100,11 +124,13 @@ export function settleClaudeUnexpectedExit( failure: startupAcquisitionRefusal(exit.error) || providerSpawnAbsent(exit.error) ? providerStartupFailureFact(exit.error) - : providerExitObserved(exit.error) - ? agentSessionFailureFact('providerExited', { - detail: providerDiagnosticOf(exit.error) - }) - : agentSessionFailureFact('hostFault'), + : startupFailure + ? providerStartupFailureFact(startupFailure) + : providerExitObserved(exit.error) + ? agentSessionFailureFact('providerExited', { + detail: providerDiagnosticOf(exit.error) + }) + : agentSessionFailureFact('hostFault'), cause: 'unexpected-exit', ...(report instanceof AgentSessionAcquisitionRootExitObservedError && report.deathEvidenceKind === 'kill-delivered' diff --git a/src/main/claude/structured/claude-structured-session-startup-exit.ts b/src/main/claude/structured/claude-structured-session-startup-exit.ts new file mode 100644 index 000000000000..76007297049a --- /dev/null +++ b/src/main/claude/structured/claude-structured-session-startup-exit.ts @@ -0,0 +1,73 @@ +import { isClaudeQueryClosed } from '../claude-agent-sdk-control-requests' +import { withTimeout } from '../../../shared/promise-timeout-fallback' +import { + providerExitObserved, + startupAcquisitionRefusal, + withObservedProviderExit +} from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import type { ClaudeStreamJsonConnection } from '../claude-stream-json-connection' +import type { StructuredAgentSessionEventSink } from '../../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { claudeStartFailure, claudeStartupAnswer } from '../claude-startup-failure' +import type { AgentSessionStartFailure } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' + +/** A physical exit waits briefly for initialize's answer, which the start's verdict weighs + * against the exit; a query closed by that exit is no answer. */ +export async function claudeStartupAnswerAfterExit( + connection: Pick, + control: Promise, + exit: unknown +): Promise { + const outcome = await withTimeout( + control.then( + () => undefined, + (error: unknown) => error + ), + 2_000, + undefined + ) + return claudeStartFailure([ + connection.startupFailure?.(), + isClaudeQueryClosed(outcome) ? undefined : outcome, + exit + ]) +} + +/** Orca's own record of the exit an SDK report describes, which carries what ended it, such as a + * missing CLI; the start's verdict weighs it against the report. */ +export async function claudeExitForReport( + connection: Pick, + error: unknown +): Promise { + return error instanceof Error && claudeStartupAnswer(error) === 'reported' && connection.exited + ? withTimeout(connection.exited, 2_000, undefined) + : undefined +} + +/** An exit before publication can follow initialize's answer, which only publication reads. */ +export async function claudeStartupError( + error: unknown, + connection: ClaudeStreamJsonConnection | null +): Promise { + const refusal = startupAcquisitionRefusal(error) + if (refusal || !connection) { + return refusal ?? error + } + if (providerExitObserved(error)) { + return claudeStartupAnswerAfterExit(connection, connection.initializationResult(), error) + } + const result = connection.startupFailure?.() + return result ? claudeStartFailure([result, error]) : error +} + +/** An unexpected exit, and when it ends the start: a delivered initialize answer outranks it. The + * start's root-exit receipt is taken now, before the answer join can delay it. */ +export function claudeStartupExit( + error: Error, + connection: ClaudeStreamJsonConnection | null, + events?: Pick +): [exited: Error, ended: Promise] { + events?.observeRootExit?.() + // Mark in place because the fault report may retain this error object. + const exited = withObservedProviderExit(error) + return [exited, claudeStartupError(exited, connection)] +} diff --git a/src/main/claude/structured/claude-structured-session-startup-failure.test.ts b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts new file mode 100644 index 000000000000..e80f775ee6dc --- /dev/null +++ b/src/main/claude/structured/claude-structured-session-startup-failure.test.ts @@ -0,0 +1,422 @@ +import sdkExit from '../fixtures/startup-sdk-process-exit.json' +import { afterEach, expect, it, vi } from 'vitest' +import { findInErrorChain } from '../../provider-process/provider-error-chain' +import { withMissingProviderExecutable } from '../../provider-process/provider-executable-missing' +import { claudeStartupAnswerObservation } from '../claude-startup-answer-observation' +import { runClaudeControl } from '../claude-agent-sdk-control-requests' +import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { + PROVIDER_SESSION_ID, + adapterAtPublishFor, + adapterFor, + fakeClaude, + identityFor, + recordingJournalSink, + tick +} from './claude-structured-session-test-support' +import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' +import { ClaudeStructuredSessionAdapter } from './claude-structured-session-adapter' +import { NDJSON_MAX_LINE_BYTES } from '../../../shared/main-process-ndjson-framer' +import { claudeAcquireFailure, claudeStartFailure } from '../claude-startup-failure' +import { retryableAgentSessionAcquisition } from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' + +const retryableStructuredAgentSessionStart = (error: unknown): boolean => + claudeStartFailure(error).retry === 'safe' +const retryableStructuredAgentSessionAcquisition = (error: unknown): boolean => + retryableAgentSessionAcquisition(claudeAcquireFailure(error)) + +afterEach(() => vi.useRealTimers()) + +it.each(['control_request_failed', 'process_exited_nonzero'])( + 'preserves the independently pending SDK %s outcome when exit proof wins first', + async (errorClass) => { + vi.useFakeTimers() + const claude = fakeClaude({ holdInit: true }) + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterAtPublishFor(claude, {}, events) + const acquired = await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'race' + }) + const connection = claude.connections[0] + const pending = Promise.withResolvers() + connection.initializationResult = () => + runClaudeControl('initialize', () => pending.promise, null) + // Delay SDK cleanup, independently of the actual process exit. + connection.close = () => new Promise(() => {}) + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + const exit = withObservedProviderExit(new Error('claude exited')) + connection.handlers.onExit?.(exit, { expected: false }) + await vi.advanceTimersByTimeAsync(10) + const recorded = sdkExit.records[0].initializationError + if (!recorded) { + throw new Error('missing recorded SDK exit') + } + pending.reject( + errorClass === 'process_exited_nonzero' + ? Object.assign(new Error(recorded.message), recorded) + : Object.assign(new Error('SDK terminal outcome'), { errorClass, exitCode: 1 }) + ) + await vi.advanceTimersByTimeAsync(2_000) + const error = await result + // The SDK's report of the exit is no answer, so Orca's own exit record decides the start. + if (errorClass === 'process_exited_nonzero') { + expect(error.cause).toBe(exit) + } else { + expect(error.cause.cause).toMatchObject({ errorClass, exitCode: 1 }) + } + expect(retryableStructuredAgentSessionStart(error)).toBe( + errorClass === 'process_exited_nonzero' + ) + expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) + claude.dispose() + await adapter.closeAll() + } +) + +it('retains a failed result before a stalled SDK transcript flush and never waits on startup cursor persistence', async () => { + vi.useFakeTimers() + const claude = fakeClaude({ holdInit: true }) + const persist = vi.fn(() => new Promise(() => {})) + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterAtPublishFor(claude, {}, events, [], undefined, persist) + const acquired = await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'flush' }) + const connection = claude.connections[0] + connection.close = () => new Promise(() => {}) + const observation = claudeStartupAnswerObservation() + connection.startupFailure = observation.failure + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + observation.feed( + `${JSON.stringify({ + type: 'result', + is_error: true, + subtype: 'error_during_execution', + errors: ['startup refused'] + })}\n` + ) + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(withObservedProviderExit(new Error('exited')), { expected: false }) + await vi.advanceTimersByTimeAsync(2_100) + const error = await result + expect(error.cause.errorClass).toBe('error_result') + expect(retryableStructuredAgentSessionStart(error)).toBe(false) + expect(persist).not.toHaveBeenCalled() + expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) + claude.dispose() + await adapter.closeAll() +}) + +it('treats a turn that settled without an Orca dispatch as past startup on exit', async () => { + const claude = fakeClaude() + const persistedHandles: unknown[] = [] + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterFor(claude, {}, events, persistedHandles) + await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'undispatched', + events: { ...recordingJournalSink(), written: async () => ({ ok: true }) } + }) + const connection = claude.connections[0] + // No send ran, so the startup observer still holds a later failed result. + connection.startupFailure = () => + Object.assign(new Error('claude reported a failed result during startup'), { + errorClass: 'error_result' + }) + const frame = (message: Record) => + connection.handlers.onMessage?.({ session_id: PROVIDER_SESSION_ID, ...message }) + frame({ + type: 'assistant', + uuid: 'undispatched-leaf', + message: { role: 'assistant', content: [{ type: 'text', text: 'answer' }] } + }) + frame({ type: 'result' }) + connection.handlers.onExit?.(withObservedProviderExit(new Error('exited')), { expected: false }) + await tick() + + expect(persistedHandles).toContainEqual( + expect.objectContaining({ leafUuid: 'undispatched-leaf' }) + ) + expect(events.at(-1)).toMatchObject({ type: 'ended', failure: { kind: 'providerExited' } }) + claude.dispose() + await adapter.closeAll() +}) + +it.each(['the SDK reports the exit first', "Orca's own exit arrives first"])( + "reads a missing CLI from Orca's exit when %s", + async (order) => { + const claude = fakeClaude({ holdInit: true }) + const adapter = adapterAtPublishFor(claude) + const acquired = await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'missing-cli' + }) + const connection = claude.connections[0] + const exited = Promise.withResolvers() + connection.exited = exited.promise + const report = Promise.withResolvers() + connection.initializationResult = () => + runClaudeControl('initialize', () => report.promise, null) + // A supervised spawn ENOENT: the supervisor exits 127 and the SDK reports only that exit. + const sdkReport = Object.assign(new Error('Claude Code process exited with code 127'), { + errorClass: 'process_exited_nonzero', + exitCode: 127 + }) + const ownExit = withMissingProviderExecutable( + withObservedProviderExit( + new Error('claude stream-json exited (code 127): spawn claude ENOENT') + ) + ) + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + await tick() + if (order === 'the SDK reports the exit first') { + report.reject(sdkReport) + await tick() + exited.resolve(ownExit) + } else { + exited.resolve(ownExit) + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(ownExit, { expected: false }) + await tick() + report.reject(sdkReport) + } + const error = await result + expect(error).toMatchObject({ retry: 'never', fact: { kind: 'cliMissing' } }) + // The verdict keeps Orca's record, diagnostic and absence marker, not the SDK's summary. + expect(findInErrorChain(error, (cause) => (cause === ownExit ? true : undefined))).toBe(true) + claude.dispose() + await adapter.closeAll() + } +) + +it('settles an unanswered Claude startup exit with the same bounded recovery eligibility', async () => { + vi.useFakeTimers() + const claude = fakeClaude({ holdInit: true }) + const adapter = adapterAtPublishFor(claude) + const acquired = await adapter.acquire({ + identity: identityFor(), + fence: 7, + spawnToken: 'no-answer' + }) + const connection = claude.connections[0] + connection.close = () => new Promise(() => {}) + const result = adapter + .open({ + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: acquired.acquisitionGeneration, + signal: new AbortController().signal + }) + .catch((error) => error) + const error = withObservedProviderExit(new Error('child exited before initialize')) + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(error, { expected: false }) + await vi.advanceTimersByTimeAsync(2_100) + expect((await result).cause).toBe(error) + expect(retryableStructuredAgentSessionStart(await result)).toBe(true) + claude.dispose() + await adapter.closeAll() +}) + +it('keeps a rejected initialize final when the child exits before acquisition publishes', async () => { + const claude = fakeClaude({ holdInit: true }) + const probing = Promise.withResolvers() + const startTime = Promise.withResolvers() + const adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: '/work/repo', + claudeConfigDir: '/accounts/claude', + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: null, + resumesTranscript: false, + cliFlag: async () => 'supported' as const, + continuesChain: false + }), + openConnection: claude.openConnection, + readProcessStartTime: async () => { + probing.resolve() + return startTime.promise + } + }) + const result = adapter + .acquire({ identity: identityFor(), fence: 7, spawnToken: 'prepublication' }) + .catch((error) => error) + await probing.promise + const connection = claude.connections[0] + // The SDK's answer to an error control_response for initialize, as the published path sees it. + const answer = runClaudeControl( + 'initialize', + () => + Promise.reject( + Object.assign(new Error('initialize rejected'), { errorClass: 'control_request_failed' }) + ), + null + ) + connection.initializationResult = () => answer + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'unverifiable' } + connection.close = async () => false + connection.handlers.onExit?.(withObservedProviderExit(new Error('claude exited (code 1)')), { + expected: false + }) + startTime.resolve(1_700_000_000_000) + const error = await result + expect( + findInErrorChain(error, (cause) => ('errorClass' in cause ? cause.errorClass : undefined)) + ).toBe('control_request_failed') + expect(retryableStructuredAgentSessionAcquisition(error)).toBe(false) + claude.dispose() + connection.close = async () => true + await adapter.closeAll() +}) + +it('parses only failed root results at startup, bounded by candidates rather than records', async () => { + const flush = () => new Promise((resolve) => setImmediate(resolve)) + const hook = (output: string) => + `${JSON.stringify({ + type: 'system', + subtype: 'hook_response', + hook_name: 'SessionStart:startup', + hook_event: 'SessionStart', + output, + stdout: '', + stderr: '' + })}\n` + const assistant = (text: string) => + `${JSON.stringify({ + type: 'assistant', + parent_tool_use_id: null, + message: { role: 'assistant', content: [{ type: 'text', text }] } + })}\n` + const failedResult = (subtype: string) => + `${JSON.stringify({ type: 'result', subtype, is_error: true })}\n` + // Counts the real observer's work for one child at a given startup record population. + const observeStartup = async (records: number) => { + const observation = claudeStartupAnswerObservation() + const parse = vi.spyOn(JSON, 'parse') + try { + const oversized = hook('x'.repeat(NDJSON_MAX_LINE_BYTES + 1)) + observation.feed(oversized.slice(0, NDJSON_MAX_LINE_BYTES / 2)) + observation.feed(oversized.slice(NDJSON_MAX_LINE_BYTES / 2)) + for (let index = 0; index < records; index += 1) { + observation.feed( + index % 2 === 0 + ? hook(JSON.stringify({ type: 'result', is_error: true, index })) + : assistant(`working ${index}`) + ) + } + // Later failed results coalesced into the capturing chunk must not be parsed. + observation.feed( + failedResult('error_during_execution') + + failedResult('error_in_capturing_chunk').repeat(records) + ) + await flush() + const parsed = parse.mock.calls.map(([text]) => String(text)) + const failure = observation.failure() + parse.mockClear() + observation.feed(failedResult('error_after_capture')) + observation.feed(hook('after capture')) + observation.finish() + observation.feed(failedResult('error_after_finish')) + await flush() + return { + irrelevantParses: parsed.filter( + (text) => !text.includes('error_during_execution') && !text.includes('capturing_chunk') + ).length, + candidateParses: parsed.filter((text) => text.includes('error_during_execution')).length, + sameChunkParsesAfterCapture: parsed.filter((text) => text.includes('capturing_chunk')) + .length, + failure, + parsesAfterCapture: parse.mock.calls.length + } + } finally { + parse.mockRestore() + } + } + const n = await observeStartup(100) + const tenN = await observeStartup(1000) + expect(n).toMatchObject({ + irrelevantParses: 0, + candidateParses: 1, + sameChunkParsesAfterCapture: 0, + failure: expect.objectContaining({ errorClass: 'error_result' }), + parsesAfterCapture: 0 + }) + expect(tenN).toEqual(n) +}) + +it('takes no startup answer from a line over the budget', async () => { + const observation = claudeStartupAnswerObservation() + const flush = () => new Promise((resolve) => setImmediate(resolve)) + const large = 'x'.repeat(NDJSON_MAX_LINE_BYTES + 1) + // Tool input is arbitrary JSON; a result-shaped object inside it is not the provider's answer. + const assistant = `${JSON.stringify({ + type: 'assistant', + parent_tool_use_id: null, + message: { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'tool-1', + name: 'Write', + input: { type: 'result', is_error: true, content: large } + } + ] + } + })}\n` + observation.feed(assistant.slice(0, NDJSON_MAX_LINE_BYTES)) + observation.feed(assistant.slice(NDJSON_MAX_LINE_BYTES)) + await flush() + expect(observation.failure()).toBeUndefined() + // The parent that marks a subagent result can sit past the retained prefix. + observation.feed( + `${JSON.stringify({ type: 'result', is_error: true, errors: [large], parent_tool_use_id: 'task-1' })}\n` + ) + // Intended limit: an oversized failed root result is left to the SDK answer or the unanswered exit. + observation.feed(`${JSON.stringify({ type: 'result', is_error: true, errors: [large] })}\n`) + await flush() + expect(observation.failure()).toBeUndefined() + observation.feed( + `${JSON.stringify({ type: 'result', subtype: 'error_during_execution', is_error: true })}\n` + ) + await flush() + expect(observation.failure()).toMatchObject({ errorClass: 'error_result' }) + const parse = vi.spyOn(JSON, 'parse') + try { + observation.feed(`${JSON.stringify({ type: 'result', subtype: 'later', is_error: true })}\n`) + expect(parse).not.toHaveBeenCalled() + } finally { + parse.mockRestore() + } +}) diff --git a/src/main/claude/structured/claude-structured-session-startup.test.ts b/src/main/claude/structured/claude-structured-session-startup.test.ts index 8136ddeef332..7ab47856e48f 100644 --- a/src/main/claude/structured/claude-structured-session-startup.test.ts +++ b/src/main/claude/structured/claude-structured-session-startup.test.ts @@ -197,7 +197,7 @@ describe('Claude host-awaited open', () => { // Recorded result; the orderings are constructed. A start another failure ended reports that cause. it.each([ ['initialize rejected', 1], - ['an exit with no stderr', 1], + ['an exit with no stderr', 0], ['an exit naming the missing conversation', 0], ['the same result', 0] ] as const)('a startup failure result after %s keeps %i rows', async (ended, rows) => { @@ -220,9 +220,12 @@ describe('Claude host-awaited open', () => { claudeStreamJsonExitError(stderr, { code: 1, signal: null, processless: false }) ) } - // An exit declines at once, before the buffered result drains. + // An exit naming no cause leaves the start to the buffered result, so that result is its one failure. claudeFrame(connection, { ...MISSING, session_id: PROVIDER_SESSION_ID }) - await open + const error = await open + if (ended === 'an exit with no stderr') { + expect(startupAcquisitionRefusal(error)?.reason).toBe('savedConversationMissing') + } expect(errorRows(appendItem)).toHaveLength(rows) }) @@ -285,7 +288,7 @@ describe('Claude host-awaited open', () => { connection.handlers.onFault?.(fault) claudeFrame(connection, { ...MISSING, session_id: PROVIDER_SESSION_ID }) release.resolve({ ok: true }) - expect(await open).toBe(fault) + expect(await open).toMatchObject({ cause: fault }) expect(errorRows(appendItem)).toHaveLength(rows) } ) diff --git a/src/main/claude/structured/claude-structured-session-startup.ts b/src/main/claude/structured/claude-structured-session-startup.ts index 9b524a08f88a..4b0e4024f45c 100644 --- a/src/main/claude/structured/claude-structured-session-startup.ts +++ b/src/main/claude/structured/claude-structured-session-startup.ts @@ -1,3 +1,9 @@ +import { isClaudeQueryClosed } from '../claude-agent-sdk-control-requests' +import { claudeStartFailure } from '../claude-startup-failure' +import { + startupAcquisitionRefusal, + providerExitObserved +} from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' // Initialize gates host open after process publication; settings enrichment runs independently. import type { AgentSessionAccountKind } from '../../../shared/agent-session/agent-session-availability' @@ -26,6 +32,10 @@ import { readClaudeSettingsEffort } from './launch/claude-structured-session-options' import { claudeExecutionObservations } from './claude-structured-execution-observations' +import { + claudeExitForReport, + claudeStartupAnswerAfterExit +} from './claude-structured-session-startup-exit' import type { ClaudeAuthDiagnostic, ClaudeSession, @@ -64,22 +74,41 @@ export async function readClaudeStartupFacts(input: { emit: (event: ClaudeStructuredSessionEvent) => void }): Promise { let timer: ReturnType | undefined + const control = input.connection.initializationResult().then((result) => { + const authError = claudeInitializationAuthError(result, input.account) + if (authError) { + throw authError + } + return result + }) const initialization = await Promise.race([ - input.connection - .initializationResult() - .catch((error: unknown) => { - // The SDK may replace the child's own exit diagnostic with a generic closed-query error. + control.catch(async (error: unknown) => { + let proof: unknown + try { input.initProof.seen() + } catch (proofError) { + // A refusal, or what closed the query before Claude identified itself, is an answer. + proof = + startupAcquisitionRefusal(proofError) || isClaudeQueryClosed(error) ? proofError : null + } + throw claudeStartFailure([ + input.connection.startupFailure?.(), + proof, + error, + await claudeExitForReport(input.connection, error) + ]) + }), + Promise.race([ + input.initProof.promise.then(() => new Promise(() => {})), + input.connection.exited?.then((error) => { + throw error + }) ?? new Promise(() => {}) + ]).catch(async (error: unknown) => { + if (startupAcquisitionRefusal(error) || !providerExitObserved(error)) { throw error - }) - .then((result) => { - const authError = claudeInitializationAuthError(result, input.account) - if (authError) { - throw authError - } - return result - }), - input.initProof.promise.then(() => new Promise(() => {})), + } + throw await claudeStartupAnswerAfterExit(input.connection, control, error) + }), new Promise((_resolve, reject) => { timer = setTimeout( () => @@ -97,7 +126,10 @@ export async function readClaudeStartupFacts(input: { }) ]).finally(() => clearTimeout(timer)) if (input.connection.closed) { - throw new Error('claude session closed before startup completed') + throw ( + input.connection.startupFailure?.() ?? + new Error('claude session closed before startup completed') + ) } input.emit({ type: 'options', @@ -210,7 +242,10 @@ export async function openClaudeSession(input: { const { session } = input const initialized = await input.facts if (!input.isCurrent()) { - throw new Error('claude session closed before startup completed') + throw ( + session.connection.startupFailure?.() ?? + new Error('claude session closed before startup completed') + ) } initialized.initProof.onRefusal = (error) => { if (input.isCurrent()) { diff --git a/src/main/claude/structured/claude-structured-session-state.ts b/src/main/claude/structured/claude-structured-session-state.ts index 850b3d87aba9..9479e5115dae 100644 --- a/src/main/claude/structured/claude-structured-session-state.ts +++ b/src/main/claude/structured/claude-structured-session-state.ts @@ -358,6 +358,8 @@ export async function cancelClaudeAcquisitionAttempt( } export type ClaudeAcquireCallbacks = { + /** Publishes the session, its start ending admitted first so its exit is never missed. */ + publish: (session: ClaudeSession) => void deliver: (attempt: ClaudeAcquisitionAttempt, sessionId: string, event: () => void) => void emit: ( session: ClaudeSession | null, diff --git a/src/main/claude/structured/claude-structured-start-ending.ts b/src/main/claude/structured/claude-structured-start-ending.ts new file mode 100644 index 000000000000..a685b52f6625 --- /dev/null +++ b/src/main/claude/structured/claude-structured-start-ending.ts @@ -0,0 +1,90 @@ +import { waitForPromiseWithSignal } from '../../../shared/abort-signal-reason' +import { withObservedProviderExit } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import type { + AgentSessionStartEndings, + AgentSessionStartFailure +} from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' +import type { + StructuredAgentSessionProviderOpenInput, + StructuredAgentSessionProviderOpenResult +} from '../../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' +import { openClaudeStructuredSession } from './claude-structured-session-open' +import { claudeStartFailure } from '../claude-startup-failure' +import type { ClaudeSession, ClaudeSessionExit } from './claude-structured-session-state' + +/** Publishes an acquisition's session with its start ending admitted first, so its exit is never + * missed. A failed acquire has no start owner left, so it retires exactly that admission. */ +export function claudeStartPublication( + sessions: Map, + endings: AgentSessionStartEndings, + sessionId: string +): { publish: (session: ClaudeSession) => void; failed: () => void } { + let admitted: string | undefined + return { + publish: (session) => { + admitted = session.acquisitionGeneration + endings.admit(sessionId, admitted) + sessions.set(sessionId, session) + }, + failed: () => { + if (admitted !== undefined) { + endings.retire(sessionId, admitted) + } + } + } +} + +/** Claude ended on its own: unanswered, unless what ended it was its answer or a refusal. */ +export function claudeExitEnding(error: Error): AgentSessionStartFailure { + return claudeStartFailure(withObservedProviderExit(new Error(error.message, { cause: error }))) +} + +/** A start's open answers with Claude's verdict. An exit already seen is published first, so the + * host never reads the rejection before the exit that explains it. */ +export async function openClaudeStart( + input: Omit[0], 'session'> & { + exits: Map + endings: AgentSessionStartEndings + } +): Promise { + const { request } = input + const session = input.sessions.get(request.sessionId) + if (!session) { + throw await claudeEndedStart( + input.exits.get(request.sessionId)?.session, + request, + input.endings + ) + } + try { + return await openClaudeStructuredSession({ ...input, session }) + } catch (error) { + const exit = input.exits.get(request.sessionId) + if (exit?.session.acquisitionGeneration === request.acquisitionGeneration && exit.publication) { + await waitForPromiseWithSignal(exit.publication, request.signal) + } + throw claudeStartFailure(error) + } +} + +/** A start that ended before its open answers with its own ending: the cause its startup settled + * on once the exit's initialize join finished, else the ending taken at its exit. */ +async function claudeEndedStart( + exited: ClaudeSession | undefined, + request: StructuredAgentSessionProviderOpenInput, + endings: AgentSessionStartEndings +): Promise { + const { sessionId, acquisitionGeneration, signal } = request + const { startup } = exited?.initialization ?? {} + if (exited?.acquisitionGeneration === acquisitionGeneration && startup && !startup.opened()) { + const ended = await waitForPromiseWithSignal( + startup.race(new Promise(() => {}), signal).catch((error: unknown) => error), + signal + ) + return claudeStartFailure(ended) + } + return ( + endings.get(sessionId, acquisitionGeneration) ?? + claudeStartFailure(new Error(`no claude session for ${sessionId}`)) + ) +} diff --git a/src/main/claude/structured/claude-structured-startup-outcome.ts b/src/main/claude/structured/claude-structured-startup-outcome.ts index fc095d22ba30..3a73f80d2320 100644 --- a/src/main/claude/structured/claude-structured-startup-outcome.ts +++ b/src/main/claude/structured/claude-structured-startup-outcome.ts @@ -1,4 +1,7 @@ -import { startupAcquisitionRefusal } from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' +import { + providerExitObserved, + startupAcquisitionRefusal +} from '../../native-chat/agent-session-wire/structured-agent-session-failure-text' /** One acquisition's startup: its first ending cause is the one its open rejects with. */ export type ClaudeStartupOutcome = { @@ -10,6 +13,13 @@ export type ClaudeStartupOutcome = { race: (work: Promise, signal: AbortSignal) => Promise /** Throws the cause that ended this start; otherwise later failures belong to the journal for good. */ open: () => void + /** Whether the whole host-awaited open has returned for this start. */ + opened: () => boolean +} + +/** A fault ends the start now unless it is a proven physical exit, which onExit's initialize join settles. */ +export function endsClaudeStartup(error: unknown): boolean { + return startupAcquisitionRefusal(error) !== undefined || !providerExitObserved(error) } export function createClaudeStartupOutcome(): ClaudeStartupOutcome { @@ -60,6 +70,7 @@ export function createClaudeStartupOutcome(): ClaudeStartupOutcome { throw ended() } state = 'opened' - } + }, + opened: () => state === 'opened' } } diff --git a/src/main/codex/app-server/codex-app-server-connection-exit-order.test.ts b/src/main/codex/app-server/codex-app-server-connection-exit-order.test.ts index 6a863610562f..91ab1fe9d808 100644 --- a/src/main/codex/app-server/codex-app-server-connection-exit-order.test.ts +++ b/src/main/codex/app-server/codex-app-server-connection-exit-order.test.ts @@ -3,6 +3,9 @@ import { PassThrough } from 'node:stream' import { describe, expect, it, vi } from 'vitest' import type { spawnProcess } from '@orca/process-host' import { openCodexAppServerConnection } from './codex-app-server-connection' +import { CodexAppServerRequestError } from './codex-app-server-request-error' +import { codexStartFailure } from './codex-startup-outcome' +import canonicalRefusal from './fixtures/start-resume-canonical-filename-refusal.json' function stubChild() { const child = Object.assign(new EventEmitter(), { @@ -43,4 +46,37 @@ describe('Codex app-server exit order', () => { expect(exits[0]).toContain('codex crashed') await connection.close() }) + + it('settles a start with the final answer Codex wrote just before exiting', async () => { + const { child, spawnImpl } = stubChild() + const connection = await openCodexAppServerConnection( + { command: 'codex', args: ['app-server'] }, + {}, + spawnImpl + ) + child.stdin.on('data', (chunk: Buffer) => { + const asked = chunk + .toString() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) + .find((frame) => frame.method === canonicalRefusal.method) + if (!asked) { + return + } + const { id } = asked + // The exit is seen first; the answer's last chunk reaches Orca only afterwards. + child.emit('exit', 1, null) + setImmediate(() => + child.stdout.write( + `${JSON.stringify({ id, error: { code: canonicalRefusal.code, message: canonicalRefusal.message } })}\n` + ) + ) + }) + const resumed = connection.request(canonicalRefusal.method, {}).catch((error: unknown) => error) + const failure = codexStartFailure(await resumed) + expect(failure.cause).toBeInstanceOf(CodexAppServerRequestError) + expect(failure.retry).toBe('never') + await connection.close() + }) }) diff --git a/src/main/codex/app-server/codex-app-server-connection.ts b/src/main/codex/app-server/codex-app-server-connection.ts index 23bbb1b42cbb..614c70d35db6 100644 --- a/src/main/codex/app-server/codex-app-server-connection.ts +++ b/src/main/codex/app-server/codex-app-server-connection.ts @@ -5,6 +5,7 @@ import { createClaudeChildTreeReaper } from '../../claude/claude-agent-sdk-exit- import type { ProviderProcessLaunch } from '../../provider-process/provider-process-launch' import { buildCodexAppServerExitError } from './codex-app-server-exit-error' import { withProviderSpawnAbsence } from '../../provider-process/provider-executable-missing' +import { afterProviderOutputDrains } from '../../provider-process/provider-exit-output-drain' import { initializeCodexAppServerConnection } from './codex-app-server-handshake' import { CodexAppServerHandshakeExitUnprovenError } from './codex-app-server-handshake-exit-proof' import { @@ -48,8 +49,8 @@ async function connectCodexAppServer( // Several listeners can report one failure; preserve its first, specific cause. let terminalError: Error | null = null - function buildExitError(cause?: Error): Error { - const error = buildCodexAppServerExitError(managed.stderrTail(), cause) + function buildExitError(cause?: Error, protocolFailure = false): Error { + const error = buildCodexAppServerExitError(managed.stderrTail(), cause, protocolFailure) return withProviderSpawnAbsence(error, managed.spawnAbsence) } @@ -57,16 +58,22 @@ async function connectCodexAppServer( handlers, writeResponse, onProtocolFailure: (error) => { - handleUnexpectedEnd(error) + handleUnexpectedEnd(error, true) void managed.close() } }) - // Fail pending calls immediately, but notify the owner only once root exit is proven. - function handleUnexpectedEnd(cause?: Error): void { + // Pending calls fail once output drained, so an answer written just before exit still settles + // its call; the owner is notified only once root exit is proven. + function handleUnexpectedEnd(cause?: Error, protocolFailure = false): void { if (!terminalError) { - terminalError = buildExitError(cause) - dispatcher.failPending(terminalError) + const error = buildExitError(cause, protocolFailure) + terminalError = error + if (protocolFailure) { + dispatcher.failPending(error) + } else { + afterProviderOutputDrains(child.stdout, () => dispatcher.failPending(error), child) + } } if (managed.rootVerdict === 'exited' && closesInFlight === 0 && !exitReported) { exitReported = true @@ -105,7 +112,7 @@ async function connectCodexAppServer( } }, onFatal: (error) => { - handleUnexpectedEnd(error) + handleUnexpectedEnd(error, true) void managed.close() } }) diff --git a/src/main/codex/app-server/codex-app-server-exit-error.ts b/src/main/codex/app-server/codex-app-server-exit-error.ts index f4e04cdc936f..edb95b3ddf8b 100644 --- a/src/main/codex/app-server/codex-app-server-exit-error.ts +++ b/src/main/codex/app-server/codex-app-server-exit-error.ts @@ -9,7 +9,22 @@ import { providerStderrForDisplay } from '../../provider-process/provider-spawn- const EXIT_DETAIL_MAX_CHARS = 400 -export function buildCodexAppServerExitError(stderrTail: string, cause?: Error): Error { +/** The transport ended. Protocol failures are Orca refusing what Codex wrote, so not an exit. */ +export class CodexAppServerExitError extends Error { + constructor( + message: string, + readonly protocolFailure = false + ) { + super(message) + this.name = 'CodexAppServerExitError' + } +} + +export function buildCodexAppServerExitError( + stderrTail: string, + cause?: Error, + protocolFailure = false +): Error { const tail = providerStderrForDisplay(stderrTail).trim().slice(0, EXIT_DETAIL_MAX_CHARS) // The tail is Codex's own stderr: a log, kept behind Details. const diagnostic = providerDiagnostic(tail, 'log') @@ -22,6 +37,9 @@ export function buildCodexAppServerExitError(stderrTail: string, cause?: Error): ) } const detail = cause ? `: ${cause.message}` : tail ? `: ${tail}` : '' - const error = new Error(`codex app-server connection ended${detail}`) + const error = new CodexAppServerExitError( + `codex app-server connection ended${detail}`, + protocolFailure + ) return cause ? error : withProviderDiagnostic(error, diagnostic) } diff --git a/src/main/codex/app-server/codex-app-server-record-dispatch.ts b/src/main/codex/app-server/codex-app-server-record-dispatch.ts index 8f595d3d2317..ef50c49d2682 100644 --- a/src/main/codex/app-server/codex-app-server-record-dispatch.ts +++ b/src/main/codex/app-server/codex-app-server-record-dispatch.ts @@ -118,7 +118,8 @@ export function createCodexAppServerRecordDispatcher(input: { waiter.method, typeof error.code === 'number' ? error.code : null, `codex app-server ${waiter.method} failed: ${detail}`, - typeof error.message === 'string' ? error.message : undefined + typeof error.message === 'string' ? error.message : undefined, + error.data ) ) return diff --git a/src/main/codex/app-server/codex-app-server-request-error.ts b/src/main/codex/app-server/codex-app-server-request-error.ts index fca7c7e1081e..42ef064901db 100644 --- a/src/main/codex/app-server/codex-app-server-request-error.ts +++ b/src/main/codex/app-server/codex-app-server-request-error.ts @@ -7,15 +7,18 @@ import { export class CodexAppServerRequestError extends Error { /** Codex's own `error.message`, kept apart from the Orca text around it. */ readonly providerDiagnostic?: ProviderDiagnostic + readonly data?: unknown constructor( readonly method: string, readonly code: number | null, message: string, - providerMessage?: string + providerMessage?: string, + data?: unknown ) { super(message) this.name = 'CodexAppServerRequestError' + Object.defineProperty(this, 'data', { value: data, enumerable: false }) const diagnostic = providerMessage === undefined ? undefined : providerDiagnostic(providerMessage, 'person') if (diagnostic) { diff --git a/src/main/codex/app-server/codex-startup-outcome.ts b/src/main/codex/app-server/codex-startup-outcome.ts new file mode 100644 index 000000000000..55b420c615c3 --- /dev/null +++ b/src/main/codex/app-server/codex-startup-outcome.ts @@ -0,0 +1,80 @@ +import { + agentSessionAcquireFailure, + agentSessionStartFailure, + type AgentSessionStartAnswer, + type AgentSessionStartFailure +} from '../../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { CodexAppServerRequestError } from './codex-app-server-request-error' +import { CodexAppServerUnsupportedError } from './codex-app-server-session' +import { CodexAppServerExitError } from './codex-app-server-exit-error' +import { CodexAppServerFrameSizeError } from './codex-app-server-frame-size-error' + +/** Codex's answer to a start request is final unless Codex declared it temporary; its transport + * ending with no answer once output drained is an unanswered start. */ +export const codexStartupAnswer: AgentSessionStartAnswer = (cause) => { + if (cause instanceof CodexAppServerRequestError) { + return temporaryCodexStartupAnswer(cause) ? 'temporary' : 'final' + } + if ( + cause instanceof CodexAppServerUnsupportedError || + cause instanceof CodexAppServerFrameSizeError + ) { + return 'final' + } + if (cause instanceof CodexAppServerExitError) { + return cause.protocolFailure ? 'final' : 'exited' + } + return undefined +} + +export function codexStartFailure(error: unknown, exited = false): AgentSessionStartFailure { + return agentSessionStartFailure(error, codexStartupAnswer, exited) +} + +export function codexAcquireFailure(error: unknown): unknown { + return agentSessionAcquireFailure(error, codexStartupAnswer) +} + +/** Only declarations whose provider contract permits replay of a start. */ +export function temporaryCodexStartupAnswer(error: CodexAppServerRequestError): boolean { + const message = error.providerDiagnostic?.text ?? error.message + if (error.code === -32001 && message === 'Server overloaded; retry later.') { + return true + } + if (error.method !== 'thread/start' && error.method !== 'thread/resume') { + return false + } + const data = typeof error.data === 'object' && error.data !== null ? error.data : {} + if (error.code === -32600) { + if ('reason' in data && data.reason === 'serverShuttingDown') { + return true + } + if (message === 'Workspace change in progress; retry when it completes') { + return true + } + if ( + error.method === 'thread/resume' && + /^thread .+ is closing; retry thread\/resume after the thread is closed$/.test(message) + ) { + return true + } + if ( + 'reason' in data && + data.reason === 'cloudConfigBundle' && + !('action' in data && data.action === 'relogin') && + 'errorCode' in data + ) { + return ( + data.errorCode === 'Timeout' || + (data.errorCode === 'RequestFailed' && + 'detail' in data && + data.detail === 'Failed to load cloud config bundle (workspace-managed policies).') + ) + } + } + return ( + error.method === 'thread/start' && + error.code === -32603 && + /^error creating thread: .*thread-store retryable error: /s.test(message) + ) +} diff --git a/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json b/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json new file mode 100644 index 000000000000..ac7884786668 --- /dev/null +++ b/src/main/codex/app-server/fixtures/start-resume-canonical-filename-refusal.json @@ -0,0 +1,6 @@ +{ + "capture": "Wire response from codex-cli 0.166.0 thread/resume captured in an isolated QA home on 2026-11-02; paths sanitized; the response has no error.data", + "method": "thread/resume", + "code": -32600, + "message": "paginated rollout path `/qa-home/userdata/codex-runtime-home/home/sessions/2026/11/01/rollout-2026-11-01T20-18-12.123Z-01a19a7a-a7b8-7790-9e1d-73d9040a7f14.jsonl` does not have a canonical rollout filename" +} diff --git a/src/main/codex/codex-scripted-child.test-fixture.ts b/src/main/codex/codex-scripted-child.test-fixture.ts index 5d40a7a41751..c49ea26d4d18 100644 --- a/src/main/codex/codex-scripted-child.test-fixture.ts +++ b/src/main/codex/codex-scripted-child.test-fixture.ts @@ -1,7 +1,12 @@ +import { + providerDiagnostic, + withProviderDiagnostic +} from '../../shared/agent-session/failures/agent-session-failure' import type { ScriptedAgentChild } from '../runtime/structured-agent-scripted-child.test-fixture' import type { CodexAppServerConnection } from './app-server/codex-app-server-connection-types' import type { openCodexAppServerConnection } from './app-server/codex-app-server-connection' import { CodexAppServerRequestError } from './app-server/codex-app-server-request-error' +import { CodexAppServerExitError } from './app-server/codex-app-server-exit-error' import { CodexAppServerTimeoutError } from './app-server/codex-app-server-session' import { codexTurnLifecycleFake } from './codex-turn-lifecycle-fake' import { withProviderRequestTimeout } from '../provider-process/provider-request-timeout' @@ -55,6 +60,7 @@ export const codexScriptedChild = ( let closed = 0 const prompts: string[] = [] const requests: { method: string; params?: Record }[] = [] + let exitNewest: ((message: string) => void) | undefined let newest: ReturnType> | undefined const open: typeof openCodexAppServerConnection = async (_launch, handlers = {}) => { notify = (method, params) => handlers.onNotification?.(method, params) @@ -153,6 +159,16 @@ export const codexScriptedChild = ( return true } } + exitNewest = (message) => { + ended = true + // As the real connection: an exit fails the pending calls with its own exit error. + const error = withProviderDiagnostic( + new CodexAppServerExitError(message), + providerDiagnostic(message, 'log') + ) + gate.reject(error) + handlers.onExit?.(error, { expected: false }) + } // A gate can be closed before initialization starts or after it has already answered. void gate.promise.catch(() => {}) await handlers.onSpawned?.(connection.pid ?? 0) @@ -185,6 +201,7 @@ export const codexScriptedChild = ( completeTurns = value }, releaseHandshake: () => newest?.resolve({}), + exitHandshake: (message) => exitNewest?.(message), failHandshake: (message) => newest?.reject(new CodexAppServerRequestError('initialize', -32603, message, message)), prompts: () => prompts, diff --git a/src/main/codex/codex-structured-session-acquire.ts b/src/main/codex/codex-structured-session-acquire.ts index 7d95b68a0641..82d1d687af3d 100644 --- a/src/main/codex/codex-structured-session-acquire.ts +++ b/src/main/codex/codex-structured-session-acquire.ts @@ -40,6 +40,7 @@ import type { CodexStructuredSessionTeardown } from './codex-structured-session- import type { CodexStructuredNotificationRetry } from './codex-structured-notification-retry' import type { deliverCodexServerRequest } from './codex-structured-provider-events' import { codexAcquisitionNotificationHandler } from './codex-structured-acquisition-notification' +import { codexStartFailure } from './app-server/codex-startup-outcome' export async function acquireCodexStructuredSession(input: { input: StructuredAgentSessionAcquireInput @@ -76,6 +77,9 @@ export async function acquireCodexStructuredSession(input: { let account: AgentSessionAccountKind | undefined // Frame row identities include this generation. const acquisitionGeneration = mintCodexAcquisitionGeneration(deps) + acquisitions.endings.admit(sessionId, acquisitionGeneration) + // Codex ending on its own before acquire returns; acquire fails with it, not a generic error. + let exitedOnItsOwn: Error | undefined const translator = createCodexSessionJournalTranslator({ sink: acquireInput.events, account: () => account, @@ -163,6 +167,17 @@ export async function acquireCodexStructuredSession(input: { onExit: (error, exit) => { acquisition.appliedSettings.end() try { + // Orca's own close or cancel ends no start; Codex ending on its own does, from the + // moment its process is seen, before spawn bookkeeping finishes. + if (!exit?.expected && !attempt.cancelled) { + exitedOnItsOwn ??= error + acquireInput.events?.observeRootExit?.() + acquisitions.endings.record( + sessionId, + acquisitionGeneration, + codexStartFailure(error) + ) + } if (attempt.acquisitionGeneration && !sessions.has(sessionId)) { attempt.exitProven = true attempt.exitError = error @@ -222,6 +237,9 @@ export async function acquireCodexStructuredSession(input: { acquisitions.assertCurrent(sessionId, attempt) const process = await spawnIdentity.read(connection.pid) acquisitions.assertCurrent(sessionId, attempt) + if (connection.closed && exitedOnItsOwn) { + throw exitedOnItsOwn + } assertCodexConnectionOpen(connection, sessionId) attempt.acquisitionGeneration = acquisitionGeneration let disposed = false @@ -259,6 +277,7 @@ export async function acquireCodexStructuredSession(input: { ) return { process, acquisitionGeneration } } catch (error) { + acquisitions.endings.retire(sessionId, acquisitionGeneration) if (sessions.get(sessionId)?.connection !== acquisition.connection) { return closeFailedCodexAcquisition({ sessionId, diff --git a/src/main/codex/codex-structured-session-adapter-fixture.ts b/src/main/codex/codex-structured-session-adapter-fixture.ts index 7e1738d7aca3..29017e372bcf 100644 --- a/src/main/codex/codex-structured-session-adapter-fixture.ts +++ b/src/main/codex/codex-structured-session-adapter-fixture.ts @@ -21,6 +21,7 @@ import type { } from './codex-structured-session-state' import { codexProviderHandle } from '../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' +import { withObservedProviderExit } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' const startLinks = new WeakMap< CodexStructuredSessionAdapter, @@ -90,13 +91,34 @@ type FakeConnection = Omit & { closeCount: number } +const STARTUP_METHODS = new Set(['thread/start', 'thread/resume', 'thread/read']) + export function fakeCodex(routes: Record = {}): { connections: FakeConnection[] openConnection: typeof openCodexAppServerConnection routes: Record } { const connections: FakeConnection[] = [] - const openConnection: typeof openCodexAppServerConnection = async (launch, handlers = {}) => { + const openConnection: typeof openCodexAppServerConnection = async ( + launch, + adapterHandlers = {} + ) => { + // As the real connection: a spontaneous exit also fails the start's calls still waiting. + const waiting = new Set<(error: Error) => void>() + const handlers: CodexAppServerConnectionHandlers = { + ...adapterHandlers, + onExit: (error, exit) => { + // The real connection's own exit error reads as the agent's exit, never a refusal. + adapterHandlers.onExit?.(exit?.expected ? error : withObservedProviderExit(error), exit) + if (exit?.expected) { + return + } + for (const fail of waiting) { + fail(error) + } + waiting.clear() + } + } const connection: FakeConnection = { launch, handlers, @@ -108,7 +130,14 @@ export function fakeCodex(routes: Record = {}): { request: async (method, params) => { connection.calls.push({ method, params }) const route = routes[method] - return route ? route(params) : {} + const answer = route ? route(params) : {} + // Turn calls keep the answers their tests script; a start's calls end with the child. + if (!(answer instanceof Promise) || !STARTUP_METHODS.has(method)) { + return answer + } + const exited = new Promise((_resolve, reject) => waiting.add(reject)) + void exited.catch(() => {}) + return Promise.race([answer, exited]) }, notify: () => {}, respond: (id, result) => connection.replies.push({ id, result }), diff --git a/src/main/codex/codex-structured-session-adapter.ts b/src/main/codex/codex-structured-session-adapter.ts index 1d32b71f0d2a..e9b80be7951c 100644 --- a/src/main/codex/codex-structured-session-adapter.ts +++ b/src/main/codex/codex-structured-session-adapter.ts @@ -49,6 +49,8 @@ import { cancelCodexStructuredTurn } from './codex-structured-prompt-ownership' import { renameCodexProviderSession } from './codex-provider-session-rename' +import { codexAcquireFailure } from './app-server/codex-startup-outcome' +import { openCodexStart, recoverCodexRewindAtStart } from './codex-structured-start-verdict' export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdapter { private readonly sessions = new Map() @@ -114,15 +116,12 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap this.handleUnhandledFrame(sessionId, kind, payload), forceCloseUnexpected: (sessionId, fence, acquisitionGeneration, reason) => this.teardown.forceCloseUnexpected(sessionId, fence, acquisitionGeneration, reason) + }).catch((error: unknown) => { + throw codexAcquireFailure(error) }) - open = (input: StructuredAgentSessionProviderOpenInput) => { - const attempt = this.acquisitions.get(input.sessionId) - if (!attempt?.open) { - return Promise.reject(new Error(`no spawned codex child for session ${input.sessionId}`)) - } - return attempt.open(input) - } + open = (input: StructuredAgentSessionProviderOpenInput) => + openCodexStart(this.acquisitions, input) /** Buffers pre-publication events and drops events from superseded children. */ private deliver( @@ -287,7 +286,11 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap codexRewind.rewindCodexSession(this.session(input.sessionId), input, this.deps.requestTimeoutMs) recoverRewind: NonNullable = (input) => - codexRewind.recoverCodexRewind(this.session(input.sessionId), input, this.deps.requestTimeoutMs) + recoverCodexRewindAtStart(this.sessions, input, this.deps.requestTimeoutMs) + + startupEnding = (id: string, generation: string) => this.acquisitions.endings.get(id, generation) + retireStartup = (sessionId: string, generation: string) => + this.acquisitions.endings.retire(sessionId, generation) compact: NonNullable = (input) => compactCodexStructuredSession(this.session(input.sessionId), input, this.deps.requestTimeoutMs) diff --git a/src/main/codex/codex-structured-session-state.ts b/src/main/codex/codex-structured-session-state.ts index 5573bf8ff982..fcd108a1f1e0 100644 --- a/src/main/codex/codex-structured-session-state.ts +++ b/src/main/codex/codex-structured-session-state.ts @@ -12,6 +12,7 @@ import { cancelProcessAcquisition } from '@orca/process-host/cancel-process-acqu import type { CodexAppServerConnection } from './app-server/codex-app-server-connection-types' import type { openCodexAppServerConnection } from './app-server/codex-app-server-connection' import { CodexAcquisitionWindow } from './codex-structured-acquisition-window' +import { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' import { createCodexTurnOpenWaits, type CodexTurnOpenWaits @@ -256,6 +257,8 @@ export function createCodexAcquisitionAttempt(): CodexAcquisitionAttempt { export class CodexAcquisitionRegistry { private readonly attempts = new Map() + /** Children that ended on their own before they were ready answer for it, open included. */ + readonly endings = new AgentSessionStartEndings() private closing = false get size(): number { @@ -316,6 +319,7 @@ export class CodexAcquisitionRegistry { close(): void { this.closing = true + this.endings.clear() } } diff --git a/src/main/codex/codex-structured-session-teardown.ts b/src/main/codex/codex-structured-session-teardown.ts index 102c56d4fced..caad90e438e8 100644 --- a/src/main/codex/codex-structured-session-teardown.ts +++ b/src/main/codex/codex-structured-session-teardown.ts @@ -24,6 +24,7 @@ export class CodexStructuredSessionTeardown { constructor(private readonly deps: CodexStructuredSessionTeardownDeps) {} close = async (sessionId: string, observeClose?: AgentSessionCloseObserver): Promise => { + this.deps.acquisitions.endings.retire(sessionId) const connection = this.deps.sessions.get(sessionId)?.connection ?? this.deps.acquisitions.get(sessionId)?.window.connection diff --git a/src/main/codex/codex-structured-start-endings.test.ts b/src/main/codex/codex-structured-start-endings.test.ts new file mode 100644 index 000000000000..77747036e22e --- /dev/null +++ b/src/main/codex/codex-structured-start-endings.test.ts @@ -0,0 +1,103 @@ +import { expect, it } from 'vitest' +import { CodexAppServerExitError } from './app-server/codex-app-server-exit-error' +import { claudeAndCodexRouter } from '../native-chat/agent-session-wire/structured-agent-session-adapter-router-test-support' +import { adapterFor, fakeCodex, identityFor } from './codex-structured-session-adapter-fixture' + +it("keeps a start's ending only while that start can still read it", async () => { + const codex = fakeCodex() + const adapter = adapterFor(codex) + const exit = () => + codex.connections + .at(-1)! + .handlers.onExit?.(new CodexAppServerExitError('codex app-server connection ended'), { + expected: false + }) + const acquire = async (id: string) => { + const acquired = await adapter.acquire({ + identity: identityFor(id), + fence: 1, + spawnToken: `spawn-${id}` + }) + return { id, generation: acquired.acquisitionGeneration } + } + const retained = (starts: { id: string; generation: string }[]) => + starts.filter(({ id, generation }) => adapter.startupEnding(id, generation)) + + // Each start's own exit is readable until its owner is done, it is released, or it is closed. + const ended: { id: string; generation: string }[] = [] + for (const id of ['owner-done', 'released', 'closed']) { + ended.push(await acquire(id)) + codex.connections.at(-1)!.closed = true + exit() + } + expect(retained(ended)).toHaveLength(3) + adapter.retireStartup(ended[0].id, ended[0].generation) + await adapter.releaseAcquisition({ sessionId: ended[1].id }) + await adapter.disposeSession(ended[2].id) + expect(retained(ended)).toEqual([]) + + // A start its owner made ready leaves nothing behind when it later exits. + const ready = await acquire('ready') + adapter.retireStartup(ready.id, ready.generation) + codex.connections.at(-1)!.closed = true + exit() + expect(retained([ready])).toEqual([]) + await adapter.closeAll() +}) + +it("keeps a routed start's ending until its owner retires it, after exit dropped the route", async () => { + const codex = fakeCodex() + const adapter = adapterFor(codex) + const router = claudeAndCodexRouter({ claude: adapterFor(fakeCodex()), codex: adapter }, () => + adapter.closeAll() + ) + const exitLatest = () => { + const connection = codex.connections.at(-1)! + connection.closed = true + const exit = () => + connection.handlers.onExit?.( + new CodexAppServerExitError('codex app-server connection ended'), + { + expected: false + } + ) + exit() + return exit + } + const starts: { id: string; generation: string }[] = [] + for (let n = 0; n < 10; n++) { + const id = `routed-${n}` + const acquired = await router.acquire({ + identity: identityFor(id), + fence: 1, + spawnToken: `s-${n}` + }) + exitLatest() + // The host acknowledges the exit before the start's owner has finished reading it. + router.acknowledgeSessionRelease(id) + expect(router.startupEnding(id, acquired.acquisitionGeneration)).toBeDefined() + router.retireStartup(id, acquired.acquisitionGeneration) + starts.push({ id, generation: acquired.acquisitionGeneration }) + } + expect(starts.filter(({ id, generation }) => adapter.startupEnding(id, generation))).toEqual([]) + + // An old owner's retirement leaves the replacement's ending, and a late old exit records nothing. + const old = await router.acquire({ + identity: identityFor('replaced'), + fence: 1, + spawnToken: 'r-1' + }) + const lateOldExit = exitLatest() + router.acknowledgeSessionRelease('replaced') + const next = await router.acquire({ + identity: identityFor('replaced'), + fence: 2, + spawnToken: 'r-2' + }) + exitLatest() + router.retireStartup('replaced', old.acquisitionGeneration) + expect(router.startupEnding('replaced', next.acquisitionGeneration)).toBeDefined() + lateOldExit() + expect(router.startupEnding('replaced', old.acquisitionGeneration)).toBeUndefined() + await adapter.closeAll() +}) diff --git a/src/main/codex/codex-structured-start-verdict.ts b/src/main/codex/codex-structured-start-verdict.ts new file mode 100644 index 000000000000..075a57c50f23 --- /dev/null +++ b/src/main/codex/codex-structured-start-verdict.ts @@ -0,0 +1,45 @@ +import type { StructuredAgentSessionAdapter } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { StructuredAgentSessionProviderOpenInput } from '../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' +import { codexStartFailure } from './app-server/codex-startup-outcome' +import { recoverCodexRewind } from './codex-structured-rewind' +import { + requireLiveCodexSession, + type CodexAcquisitionRegistry, + type CodexSession +} from './codex-structured-session-state' + +/** A start's open answers with Codex's verdict; a child that already ended answers with how it + * ended, and an answer its own exit followed is still worded as that exit. */ +export function openCodexStart( + acquisitions: CodexAcquisitionRegistry, + input: StructuredAgentSessionProviderOpenInput +): ReturnType { + const attempt = acquisitions.get(input.sessionId) + const opening = attempt?.open + ? attempt.open(input) + : Promise.reject( + acquisitions.endings.get(input.sessionId, input.acquisitionGeneration) ?? + new Error(`no spawned codex child for session ${input.sessionId}`) + ) + return opening.catch((error: unknown) => { + const exited = attempt?.exitProven === true && !attempt.cancelled + throw codexStartFailure([error, exited ? attempt?.exitError : undefined], exited) + }) +} + +/** A start reads this before it is ready, so its failure carries Codex's verdict. */ +export async function recoverCodexRewindAtStart( + sessions: Map, + input: Parameters>[0], + timeoutMs?: number +): ReturnType> { + try { + return await recoverCodexRewind( + requireLiveCodexSession(sessions, input.sessionId), + input, + timeoutMs + ) + } catch (error) { + throw codexStartFailure(error) + } +} diff --git a/src/main/jsonl-rpc/agent-connection.test.ts b/src/main/jsonl-rpc/agent-connection.test.ts index c555aa90b059..bf03c2a5633e 100644 --- a/src/main/jsonl-rpc/agent-connection.test.ts +++ b/src/main/jsonl-rpc/agent-connection.test.ts @@ -146,6 +146,8 @@ describe('JSON-lines RPC process ownership', () => { const { connection, child } = fixture({ onExit, onClose }, { exitOnEnd: false }) const pending = expect(connection.request('prompt')).rejects.toThrow('stream closed') child.stdout.emit('end') + // Output ending first waits the drain bound for the exit that says how the agent ended. + await vi.advanceTimersByTimeAsync(1_000) await pending expect(connection.closed).toBe(true) expect(connection.rootVerdict).toBe('live') diff --git a/src/main/jsonl-rpc/agent-connection.ts b/src/main/jsonl-rpc/agent-connection.ts index 7c0022818424..ec1849c897b6 100644 --- a/src/main/jsonl-rpc/agent-connection.ts +++ b/src/main/jsonl-rpc/agent-connection.ts @@ -10,6 +10,10 @@ import { type ProviderProcessExit } from '../provider-process/managed-provider-process' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' +import { + afterProviderOutputDrains, + PROVIDER_EXIT_OUTPUT_DRAIN_MS +} from '../provider-process/provider-exit-output-drain' import type { ProviderProcessCloseResult } from '../provider-process/provider-process-close' import { withProviderSpawnAbsence, @@ -29,6 +33,14 @@ export type JsonlRpcAgentConnectionOptions = JsonlRpcPeerHandlers & { onExit?: (error: Error, context: { expected: boolean; exit: ProviderProcessExit }) => void } +/** The agent process ended; pending calls fail with it only once its output drained. */ +export class JsonlRpcAgentExitError extends Error { + constructor(message: string) { + super(message) + this.name = 'JsonlRpcAgentExitError' + } +} + /** The owning runtime keeps process exit evidence separate from transport closure. */ export class JsonlRpcAgentConnection { private readonly managed: ManagedProviderProcess @@ -36,6 +48,8 @@ export class JsonlRpcAgentConnection { private closing = false private finishObservedExit?: () => void private streamExitTimer?: ReturnType + private inputEndTimer?: ReturnType + private inputEnded = false constructor( launch: ProviderProcessLaunch, @@ -53,6 +67,20 @@ export class JsonlRpcAgentConnection { managed.child.stdin, { ...options, + // Output often ends just before the exit whose report says how the agent ended. + onInputEnd: (finish) => { + // Output reports its end and then its close; both are one end, with one wait. + if (this.inputEnded) { + return + } + this.inputEnded = true + if (this.closing || managed.rootVerdict === 'exited') { + finish() + } else { + this.inputEndTimer = setTimeout(finish, PROVIDER_EXIT_OUTPUT_DRAIN_MS) + this.inputEndTimer.unref() + } + }, onClose: (error) => { const fail = (): void => { if (this.closing || managed.rootVerdict === 'exited') { @@ -87,9 +115,10 @@ export class JsonlRpcAgentConnection { managed.child.stderr.on('error', this.onError) managed.onExit((exit) => { clearTimeout(this.streamExitTimer) + clearTimeout(this.inputEndTimer) const error = withProviderSpawnAbsence( withProviderDiagnostic( - new Error( + new JsonlRpcAgentExitError( exit.processless ? 'Agent process could not start' : `Agent process exited (code ${exit.code ?? 'none'}, signal ${exit.signal ?? 'none'})` @@ -99,17 +128,7 @@ export class JsonlRpcAgentConnection { managed.spawnAbsence ) this.peer.finishOnInputEnd(error) - const stdout = managed.child.stdout - let finished = false - let timer: ReturnType | undefined const finish = (): void => { - if (finished) { - return - } - finished = true - clearTimeout(timer) - stdout.removeListener('end', finish) - stdout.removeListener('close', finish) this.finishObservedExit = undefined this.peer.close(error) managed.child.removeListener('error', this.onError) @@ -119,21 +138,15 @@ export class JsonlRpcAgentConnection { this.diagnose('Agent exit observer failed') } } - this.finishObservedExit = finish - if ( - this.closing || - exit.processless || - this.peer.closed || - stdout.destroyed || - stdout.readableEnded - ) { + if (this.closing || exit.processless || this.peer.closed || this.inputEnded) { finish() } else { - stdout.once('end', finish) - stdout.once('close', finish) // A descendant can inherit stdout after the provider root exits. - timer = setTimeout(finish, 1_000) - timer.unref() + this.finishObservedExit = afterProviderOutputDrains( + managed.child.stdout, + finish, + managed.child + ) } }) } @@ -158,8 +171,9 @@ export class JsonlRpcAgentConnection { return this.managed.lastCloseResult } - onExit(listener: () => void): void { - this.managed.onExit(listener) + /** `requested`: Orca was closing the agent when it exited. */ + onExit(listener: (requested?: boolean) => void): void { + this.managed.onExit(() => listener(this.closing)) } request( @@ -186,6 +200,7 @@ export class JsonlRpcAgentConnection { close(error?: Error): Promise { this.closing = true clearTimeout(this.streamExitTimer) + clearTimeout(this.inputEndTimer) this.finishObservedExit?.() this.peer.close(error) return this.managed.close() diff --git a/src/main/jsonl-rpc/peer-reply.ts b/src/main/jsonl-rpc/peer-reply.ts new file mode 100644 index 000000000000..2fb209517f71 --- /dev/null +++ b/src/main/jsonl-rpc/peer-reply.ts @@ -0,0 +1,70 @@ +import { z } from 'zod' + +const responseSchema = z.looseObject({ + type: z.literal('response'), + id: z.string().optional(), + command: z.string(), + success: z.boolean(), + data: z.unknown().optional(), + error: z.string().optional() +}) + +export class JsonlRpcResponseError extends Error { + constructor( + readonly command: string, + message: string + ) { + super(message) + this.name = 'JsonlRpcResponseError' + } +} + +export type JsonlRpcPendingRequest = { + command: string + resolve: (value: unknown) => void + reject: (error: Error) => void + timer?: ReturnType +} + +/** Settles the outstanding call a record answers; false when the record is no reply to one. */ +export function settleJsonlRpcReply( + record: { type: string; id?: unknown }, + pending: Map, + rejectPending: (id: string, error: Error) => void +): boolean { + if ( + record.type !== 'response' && + record.type !== 'extension_ui_request' && + typeof record.id === 'string' && + pending.has(record.id) && + (Object.hasOwn(record, 'command') || Object.hasOwn(record, 'success')) + ) { + rejectPending(record.id, new Error('Invalid agent RPC response')) + return true + } + if (record.type !== 'response' || typeof record.id !== 'string') { + return false + } + const waiting = pending.get(record.id) + if (!waiting) { + return true + } + const response = responseSchema.safeParse(record) + if (!response.success || response.data.command !== waiting.command) { + rejectPending(record.id, new Error('Invalid agent RPC response')) + return true + } + pending.delete(record.id) + clearTimeout(waiting.timer) + if (response.data.success) { + waiting.resolve(response.data.data) + } else { + waiting.reject( + new JsonlRpcResponseError( + waiting.command, + response.data.error ?? `Agent rejected ${waiting.command}` + ) + ) + } + return true +} diff --git a/src/main/jsonl-rpc/peer.ts b/src/main/jsonl-rpc/peer.ts index f08656cae757..74a1005c8603 100644 --- a/src/main/jsonl-rpc/peer.ts +++ b/src/main/jsonl-rpc/peer.ts @@ -6,22 +6,16 @@ import { } from '../../shared/main-process-ndjson-framer' import { detachProviderStreamErrorHandler } from '../provider-process/provider-stdio-error-boundary' import { ProviderStdioWriteQueue } from '../provider-process/provider-stdio-write-queue' +import { ProviderOutputEnd } from '../provider-process/provider-output-end' import { jsonlRpcRequestTimeout, resolveJsonlRpcPeerOptions, type JsonlRpcPeerOptions } from './peer-limits' import { settleOversizedJsonlRecord, type JsonlRpcRequestOptions } from './peer-oversized-record' +import { settleJsonlRpcReply, type JsonlRpcPendingRequest } from './peer-reply' const recordSchema = z.looseObject({ type: z.string() }) -const responseSchema = z.looseObject({ - type: z.literal('response'), - id: z.string().optional(), - command: z.string(), - success: z.boolean(), - data: z.unknown().optional(), - error: z.string().optional() -}) export type JsonlRpcRecord = z.infer export class JsonlRpcStreamClosedError extends Error { @@ -30,15 +24,7 @@ export class JsonlRpcStreamClosedError extends Error { this.name = 'JsonlRpcStreamClosedError' } } -export class JsonlRpcResponseError extends Error { - constructor( - readonly command: string, - message: string - ) { - super(message) - this.name = 'JsonlRpcResponseError' - } -} +export { JsonlRpcResponseError } from './peer-reply' export type JsonlRpcPeerHandlers = { /** Includes id-less deferred prompt acknowledgements; dialect failures are fatal to preserve lifecycle evidence. */ onRecord?: (record: JsonlRpcRecord) => void @@ -46,29 +32,26 @@ export type JsonlRpcPeerHandlers = { onOversizedEvent?: (type: string, prefix: string) => void onClose?: (error: Error) => void onDiagnostic?: (message: string) => void -} -type PendingRequest = { - command: string - resolve: (value: unknown) => void - reject: (error: Error) => void - timer?: ReturnType + /** Holds the close at the end of the agent's output, as for its exit report; `finish` closes. */ + onInputEnd?: (finish: () => void) => void } /** Transport only: a dialect owns turns, dialogs, and session handles. */ export class JsonlRpcPeer { private readonly limits: Required - private readonly pending = new Map() + private readonly pending = new Map() private readonly writer: ProviderStdioWriteQueue private readonly framer: ReturnType private nextId = 0 private nextOptionalId = 0 private terminalError?: Error private inputEndError?: Error + private readonly outputEnd: ProviderOutputEnd private readingPaused = false constructor( private readonly input: Readable, - private readonly output: Writable, + output: Writable, private readonly handlers: JsonlRpcPeerHandlers = {}, options: JsonlRpcPeerOptions = {} ) { @@ -76,7 +59,7 @@ export class JsonlRpcPeer { this.writer = new ProviderStdioWriteQueue( output, this.limits.maxQueuedWriteBytes, - (error) => this.close(error), + (error) => this.endOutput(error), { capacity: () => new Error('JSON-lines RPC write queue capacity exceeded'), closed: () => new Error('JSON-lines RPC output is not writable') @@ -111,11 +94,15 @@ export class JsonlRpcPeer { input.on('end', this.onEnd) input.on('close', this.onEnd) input.on('error', this.onError) - output.on('finish', this.onEnd) - output.on('close', this.onEnd) - output.on('error', this.onError) - if (input.destroyed || input.readableEnded || output.destroyed || !output.writable) { + this.outputEnd = new ProviderOutputEnd( + output, + (error) => this.endOutput(error), + () => this.inputEndError ?? new JsonlRpcStreamClosedError() + ) + if (input.destroyed || input.readableEnded) { this.onEnd() + } else if (output.destroyed || !output.writable) { + this.endOutput(new JsonlRpcStreamClosedError()) } } @@ -153,8 +140,8 @@ export class JsonlRpcPeer { params: Record = {}, options: JsonlRpcRequestOptions = {} ): Promise { - if (this.terminalError) { - return Promise.reject(this.terminalError) + if (this.terminalError ?? this.outputEnd.error) { + return Promise.reject(this.terminalError ?? this.outputEnd.error) } if (this.pending.size >= this.limits.maxPendingRequests) { return Promise.reject(new Error('JSON-lines RPC pending request capacity exceeded')) @@ -190,7 +177,10 @@ export class JsonlRpcPeer { this.pending.set(id, { command, resolve, reject, timer }) void this.send({ ...params, type: command, id }, controller.signal).catch( (error: unknown) => { - this.rejectPending(id, error instanceof Error ? error : new Error(String(error))) + // A write lost to the agent no longer reading leaves the call to the output drain. + if (!this.outputEnd.error) { + this.rejectPending(id, error instanceof Error ? error : new Error(String(error))) + } } ) }) @@ -217,9 +207,7 @@ export class JsonlRpcPeer { this.input.removeListener('end', this.onEnd) this.input.removeListener('close', this.onEnd) detachProviderStreamErrorHandler(this.input, this.onError) - this.output.removeListener('finish', this.onEnd) - this.output.removeListener('close', this.onEnd) - detachProviderStreamErrorHandler(this.output, this.onError) + this.outputEnd.detach() this.framer.reset() this.writer.close(error) for (const id of this.pending.keys()) { @@ -239,8 +227,25 @@ export class JsonlRpcPeer { this.close(error instanceof Error ? error : new Error(String(error))) } } - private readonly onEnd = (): void => - this.close(this.inputEndError ?? new JsonlRpcStreamClosedError()) + private readonly onEnd = (): void => { + const finish = (): void => + this.close(this.inputEndError ?? this.outputEnd.error ?? new JsonlRpcStreamClosedError()) + if (this.inputEndError || !this.handlers.onInputEnd) { + finish() + } else { + this.handlers.onInputEnd(finish) + } + } + + private endOutput(error: Error): void { + if (this.terminalError || !this.outputEnd.begin(error)) { + return + } + this.writer.close(error) + this.outputEnd.settle(this.pending.size > 0, this.input, () => + this.close(this.inputEndError ?? error) + ) + } private readonly onError = (error: Error): void => this.close(error) private diagnose(message: string): void { @@ -275,37 +280,9 @@ export class JsonlRpcPeer { return } const record = parsed.data - if ( - record.type !== 'response' && - record.type !== 'extension_ui_request' && - typeof record.id === 'string' && - this.pending.has(record.id) && - (Object.hasOwn(record, 'command') || Object.hasOwn(record, 'success')) - ) { - this.rejectPending(record.id, new Error('Invalid agent RPC response')) - return - } - if (record.type === 'response' && typeof record.id === 'string') { - const pending = this.pending.get(record.id) - if (!pending) { - return - } - const response = responseSchema.safeParse(record) - if (!response.success || response.data.command !== pending.command) { - this.rejectPending(record.id, new Error('Invalid agent RPC response')) - return - } - this.pending.delete(record.id) - clearTimeout(pending.timer) - if (response.data.success) { - pending.resolve(response.data.data) - } else { - pending.reject( - new JsonlRpcResponseError( - pending.command, - response.data.error ?? `Agent rejected ${pending.command}` - ) - ) + if (settleJsonlRpcReply(record, this.pending, (id, error) => this.rejectPending(id, error))) { + if (this.outputEnd.error && this.pending.size === 0) { + this.close(this.inputEndError ?? this.outputEnd.error) } return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts index 56a2afdfbbd1..5574da387daf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts @@ -162,6 +162,16 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi this.liveOwnerOrNull(sessionId)?.holdsLiveProviderProcess?.(sessionId, acquisitionGeneration) ?? false + // A start's owner outlives its route: exit acknowledgment may drop the route while the owner + // still reads its ending, so both calls reach the adapters by exact generation, not by route. + startupEnding = (sessionId: string, acquisitionGeneration: string) => + this.adapters() + .map((adapter) => adapter.startupEnding?.(sessionId, acquisitionGeneration)) + .find((ending) => ending !== undefined) + + retireStartup = (sessionId: string, acquisitionGeneration: string): void => + this.adapters().forEach((adapter) => adapter.retireStartup?.(sessionId, acquisitionGeneration)) + awaitStoppedRequestEnd = async (sessionId: string, stoppedAt: number) => this.liveOwnerOrNull(sessionId)?.awaitStoppedRequestEnd?.(sessionId, stoppedAt) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index 2db845d875b4..4a6e5c9a9d6f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -46,6 +46,7 @@ import type { StructuredAgentSessionAcquisitionReceipt } from './structured-agent-session-startup-attempt-contract' import type { AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' import type { StructuredAgentSessionProviderOpen, StructuredAgentSessionProviderOpenResult @@ -277,6 +278,13 @@ export type StructuredAgentSessionAdapter = StructuredAgentSessionAdapterStop & heldSends?(sessionId: string): ReadonlySet | undefined /** Prove this acquisitionโ€™s root alive for lease renewal; absent/false falls back to PID probing. */ holdsLiveProviderProcess?(sessionId: string, acquisitionGeneration: string): boolean + /** This exact acquisition's ending before the host committed it ready, as the adapter reads it. */ + startupEnding?( + sessionId: string, + acquisitionGeneration: string + ): AgentSessionStartFailure | undefined + /** The start's owner finished: nothing reads this acquisition's ending any more. */ + retireStartup?(sessionId: string, acquisitionGeneration: string): void /** Undefined preserves the client catalog when the provider reports none. */ readCommands?(sessionId: string): AgentSessionSlashCommand[] | undefined atRestCommands?: StructuredAgentSessionAtRestCommands diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts index 83e2798c2f7f..d481cce8d6d0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts @@ -25,7 +25,7 @@ import { registerStructuredAgentSessionStart, structuredAgentSessionStartStillQueued } from './structured-agent-session-start-attempts' -import { retryableStructuredAgentSessionAcquisition } from './structured-agent-session-start-retry' +import { retryableAgentSessionAcquisition } from './structured-agent-session-start-failure' import { failedCreateRefusal } from './structured-agent-session-failed-create-refusal' import { startFailureDiagnosticOf } from './structured-agent-session-failure-text' import { @@ -227,7 +227,7 @@ function withDiagnostic( ...(aborted ? { aborted: true as const } : {}), ...(expired ? { expired: true as const } : {}), ...(argumentProblem ? { argumentProblem } : {}), - retry: retryableStructuredAgentSessionAcquisition(error) + retry: retryableAgentSessionAcquisition(error) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts index fbd1cbfbc2f3..901d0a83b587 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts @@ -31,6 +31,7 @@ import { import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { codexProviderHandle } from '../../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { structuredAgentSessionHeldStarts } from './structured-agent-session-held-start.test-support' const CALLER = { callerKey: 'client-1' } const EXIT_REASON = 'Claude Code is not signed in. Sign in with the Claude CLI' @@ -40,6 +41,7 @@ let store: AgentSessionRecordStore let host: StructuredAgentSessionHost let acquire: Mock let generation = 0 +const starts = structuredAgentSessionHeldStarts(() => host) async function liveReader() { const events: AgentSessionSubscribeEvent[] = [] @@ -117,6 +119,7 @@ function providerSink() { } beforeEach(async () => { + starts.clear() root = await mkdtemp(join(tmpdir(), 'orca-append-delivery-')) resetHostTestOperationIds() generation = 0 @@ -136,16 +139,17 @@ beforeEach(async () => { agents: NO_STRUCTURED_AGENTS, logger: createStructuredAgentSessionLogger(), store, - adapter: { + // The adapter answers its never-proved opens with its own child's exit. + adapter: starts.wrap({ open: () => new Promise(() => {}), - acquire, + acquire: starts.held(acquire), releaseAcquisition: vi.fn(async () => true), closeSession: vi.fn(async () => true), dispatch: vi.fn(async () => ({ state: 'admitted' as const })), cancelTurn: vi.fn(async () => ({ cancelled: true })), answerPrompt: vi.fn(async () => undefined), setOption: vi.fn(async () => undefined) - }, + }), journalDatabase: openTestJournalHostDatabase(root), claimKeyId: 'key-1', mintSpawnToken: () => `spawn-${generation + 1}`, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts index bfade9e4ca0a..949d7a38d122 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts @@ -68,7 +68,7 @@ export function endStructuredAgentSessionReleasedChild( const session = context.sessions.get(sessionId) const child = session?.child if (child) { - context.runtimeState.startupAttempts.childEnded(sessionId, child) + context.runtimeState.startupAttempts.childEnded(sessionId, child, { byOrca: true }) } if ( !session || diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-publish.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-publish.ts index e6fe5badf319..e39dc941c08b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-publish.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-publish.ts @@ -134,23 +134,24 @@ export async function publishAttachedChild( candidate.child ) } - // Apply exits observed during background startup after indexing the matching child. + // Apply exits observed during background startup after indexing the matching child; the open + // claims a starting child's outcome first, so the adapter answers for its own early exit. const exited = context.runtimeState.startFlights.takeExit(sessionId, candidate.child) const indexed = conversation.child + const opens = !!attempt.startup && candidate.child.phase !== 'ready' + if (opens) { + context.openProviderChild?.(sessionId, candidate.child) + } if (exited && indexed) { await context.endExitedChild(sessionId, indexed, structuredAgentSessionEndedEventExit(exited)) - } else if (attempt.startup) { - if (candidate.child.phase === 'ready') { - noteStructuredAgentSessionProviderStarted( - context.deps, - sessionId, - () => - context.sessions.get(sessionId)?.child === candidate.child && - candidate.child.catalogSource?.() !== false - ) - } else { - context.openProviderChild?.(sessionId, candidate.child) - } + } else if (attempt.startup && !opens) { + noteStructuredAgentSessionProviderStarted( + context.deps, + sessionId, + () => + context.sessions.get(sessionId)?.child === candidate.child && + candidate.child.catalogSource?.() !== false + ) } context.publishStatus?.(sessionId) return true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index e7360736956a..096bebfd69a7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -68,7 +68,7 @@ export type StructuredAgentSessionChildExitContext< /** Re-derive current work and wake queue without requiring a row; restate rebaselines moved fences. */ generationEnded: StructuredAgentSessionGenerationEnded wakeDelivery?: (sessionId: string) => void - startupAttempts?: Pick + startupAttempts?: Pick /** Retain exits between background-start spawn and publication. */ startFlights?: Pick abortBound?: (sessionId: string) => void @@ -148,9 +148,10 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< // Use the first death observation when settling the running turn. const observation = child.exitObservation?.() const observedAt = observation?.observedAt ?? exit.observedAt ?? context.now() - // Host phase classifies startup failure even if adapters omit the flag. + // Host phase classifies startup failure even if adapters omit the flag. A starting child's + // exit cancels nothing: the start's owner gives its outcome from the adapter's answer. const exitedDuringStartup = child.phase === 'starting' - if (child.phase === 'starting' || !expected) { + if (!exitedDuringStartup && !expected) { context.abortBound?.(sessionId) } // Personal Stop cancels startup; host Stop fails it; preserve the close initiator. @@ -187,7 +188,10 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< }) context.generationEnded(sessionId) const endChild = (): void => { - context.startupAttempts?.childEnded(sessionId, child) + context.startupAttempts?.childEnded(sessionId, child, { + byOrca: expected || !!close, + ...(close?.failure ? { failure: close.failure, reason: close.reason ?? exit.reason } : {}) + }) if (session.child === child) { session.child = null } @@ -258,7 +262,9 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< ? { exitedDuringStartup: { generation: child.generation, - flightOwned: context.startFlights?.ownsChild(sessionId, child) + owned: + context.startFlights?.ownsChild(sessionId, child) || + context.startupAttempts?.ownsOpen(sessionId, child) } } : {}), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts new file mode 100644 index 000000000000..517aa04d891c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-resume-exit-start.test.ts @@ -0,0 +1,102 @@ +import { expect, it, vi } from 'vitest' +import { ClaudeStructuredSessionAdapter } from '../../claude/structured/claude-structured-session-adapter' +import { + PROVIDER_SESSION_ID, + fakeClaude +} from '../../claude/structured/claude-structured-session-test-support' +import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' +import { withObservedProviderExit } from './structured-agent-session-failure-text' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' +import { STRUCTURED_AGENT_SESSION_STARTUP_CEILING_MS } from './structured-agent-session-startup-attempt-contract' +import { + accept, + closeSession, + eventually, + host, + providerOpen, + registerStartupAttemptHostTests, + restartWith, + submission +} from './structured-agent-session-startup-attempt-host-test-support' + +registerStartupAttemptHostTests() + +const STARTED_AT = 1_800_000_000_000 + +it('ends a resumed start that exits during transcript catch-up without awaiting a cursor write', async () => { + const claude = fakeClaude() + const catchUp = Promise.withResolvers() + const persisted = Promise.withResolvers() + const persistHandle = vi.fn(() => persisted.promise) + let catchingUp = false + let ended = 0 + const lifecycleTypes: (string | undefined)[] = [] + const adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: '/work/repo', + claudeConfigDir: '/accounts/claude', + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: 'saved-leaf', + resumesTranscript: true, + continuesChain: true, + cliFlag: async () => 'supported' as const + }), + openConnection: claude.openConnection, + readProcessStartTime: async () => 1_700_000_000_000, + persistHandle, + onEvent: (event) => { + if (event.type === 'ended') { + ended += 1 + const lifecycle = structuredClaudeLifecycleEvent(event) + lifecycleTypes.push(lifecycle?.type) + if (lifecycle) { + void host.handleAdapterEvent(lifecycle) + } + } + } + }) + await restartWith((input) => { + if (!input.events) { + throw new Error('expected the host event sink') + } + return adapter.acquire({ + ...input, + events: { + ...input.events, + written: () => { + catchingUp = true + return catchUp.promise + } + } + }) + }) + providerOpen.mockImplementation(adapter.open) + closeSession.mockImplementation(adapter.closeSession) + host.deps.adapter.releaseAcquisition = adapter.releaseAcquisition + vi.setSystemTime(STARTED_AT) + const id = await accept('resume queued message') + // Initialize has answered; the open still waits on the journal before catch-up. + await eventually(() => expect(catchingUp).toBe(true)) + const flight = host.collaboratorsForTests().runtimeState.startFlights.get(SESSION)! + const connection = claude.connections[0] + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(withObservedProviderExit(new Error('claude exited (code 1)')), { + expected: false + }) + await vi.advanceTimersByTimeAsync(2_100) + expect(ended).toBe(1) + expect(lifecycleTypes).toEqual(['ended']) + expect(persistHandle).not.toHaveBeenCalled() + catchUp.resolve({ ok: true }) + persisted.resolve() + await adapter.drainObservedExits() + await flight.outcome + await eventually(async () => expect((await submission(id))?.dispatchState).not.toBe('pending')) + expect(Date.now() - STARTED_AT).toBeLessThan(STRUCTURED_AGENT_SESSION_STARTUP_CEILING_MS) + claude.dispose() + await adapter.closeAll() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts index 93f2bb742d77..57e0a0dd961b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts @@ -141,9 +141,9 @@ export function registerCodexInputFinalization(current: typeof codexInputFinaliz expect(starts()).toHaveLength(0) throw new CodexAppServerRequestError( 'thread/resume', - -32000, - 'startup failed', - 'startup failed' + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' ) } codex.connections[1]?.handlers.onExit?.(new Error('Codex died before ready'), { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 5ec86550cd99..3b4e02cfdcc6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -3,7 +3,7 @@ import { cutClosingStructuredAgentSession, type StructuredAgentSessionConversationLifetimeHost } from './structured-agent-session-conversation-lifetime-host' -import { retryableStructuredAgentSessionStart } from './structured-agent-session-start-retry' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' // Open conversations at rest; never start agents for reads. Acquire serialization once because the queue is nonreentrant. export type { StructuredAgentSessionConversationLifetime } from './structured-agent-session-host-types' @@ -37,10 +37,7 @@ import { type WorkspaceSleepCloseResult } from './structured-agent-session-workspace-sleep-stop' import type { StructuredAgentSessionExpiredStartup } from './structured-agent-session-startup-attempt' -import { - providerStartupFailureFact, - structuredAgentSessionStartFailure -} from './structured-agent-session-failure-text' +import { structuredAgentSessionStartFailure } from './structured-agent-session-failure-text' import { oldestQueuedSubmission, recordStructuredAgentSessionStartFailure @@ -212,57 +209,58 @@ export function createStructuredAgentSessionConversationLifetime( expireStartup: ({ sessionId, child }: StructuredAgentSessionExpiredStartup): void => stopStartingChild(sessionId, child), stopStartingChild, + /** The open's failed start: a direct attach records its row; a still-indexed child stops. A + * replaced, stopped or closed child only retires its owner. */ failStartingChild: ( sessionId: string, failed: StructuredAgentSessionProviderChildIdentity, - error: unknown, + verdict: AgentSessionStartFailure, openMs: number ): Promise => serialize(sessionId, async () => { const session = sessions.get(sessionId) const child = session?.child - if (!session || !child || child.close || !sameProviderChild(child, failed)) { + const indexed = !!child && sameProviderChild(child, failed) + const ended = session?.lastEndedChild + const exited = + !child && !!ended && sameProviderChild(ended, failed) && ended.cause === 'exit' + if (!session || (indexed ? child?.close : !exited)) { return } - const failure = providerStartupFailureFact(error) + const failure = verdict.fact // Persisted facts cannot retain the original error; log it here. deps().logger.warn('a provider could not finish starting', { scope: 'provider-open-failed', sessionId, failure: failure.kind, ...failure.requestTimeout, + retry: verdict.retry, openMs, - error + error: verdict.cause }) - const { runtimeState: runtime } = host.context() - const owned = runtime.startFlights.ownsChild(sessionId, child) - if (owned) { - const retry = retryableStructuredAgentSessionStart(error) - runtime.startupAttempts.childFailed(sessionId, child, failure, retry) + if (!host.context().runtimeState.startFlights.ownsChild(sessionId, failed)) { + const record = deps().store.getRecord(sessionId) + const wording = structuredAgentSessionFailureWordsContext(record) + // A start whose own exit settled first keeps that exit's wording; only a request + // deadline, the more specific explanation, outranks it. The verdict alone decides retry. + const cause = exited && !failure.requestTimeout ? { exit: ended?.failure } : { failure } + await recordStructuredAgentSessionStartFailure( + { journal: session.journal, fence: failed.fence }, + { + startKey: failed.generation, + ...structuredAgentSessionStartFailure(cause, wording) + }, + oldestQueuedSubmission(session)?.clientMessageId ?? null + ).catch((writeError: unknown) => + deps().logger.warn('recording a failed provider open failed', { + scope: 'provider-open-failure', + sessionId, + error: writeError + }) + ) } - try { - if (!owned) { - const record = deps().store.getRecord(sessionId) - const wording = structuredAgentSessionFailureWordsContext(record) - await recordStructuredAgentSessionStartFailure( - { journal: session.journal, fence: child.fence }, - { - startKey: child.generation, - ...structuredAgentSessionStartFailure({ failure }, wording) - }, - oldestQueuedSubmission(session)?.clientMessageId ?? null - ).catch((writeError: unknown) => - deps().logger.warn('recording a failed provider open failed', { - scope: 'provider-open-failure', - sessionId, - error: writeError - }) - ) - } - const reason = error instanceof Error ? error.message : String(error) - await stopAgent(sessionId, { cause: 'host-stop', reason, failure }) - } finally { - runtime.startupAttempts.childEnded(sessionId, child) + if (indexed) { + await stopAgent(sessionId, { cause: 'host-stop', reason: verdict.message, failure }) } }), dispose: (): void => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index a779a7c53057..d376688b1747 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -59,8 +59,9 @@ export type StructuredAgentSessionDeadGenerationInput = { showUnexpectedExitOutcome?: boolean exitFailure?: SubmissionRejectionFact failureTextContext?: AgentSessionFailureWordsContext - /** Share generation-keyed startup failure rows with delivery-loop settlement. */ - exitedDuringStartup?: { generation: string | null; flightOwned?: boolean } + /** Share generation-keyed startup failure rows with delivery-loop settlement. `owned`: a start + * flight or the child's open still owns its outcome and row. */ + exitedDuringStartup?: { generation: string | null; owned?: boolean } /** Refine translator-unverifiable rows after observed exit proof. */ exit?: StructuredAgentSessionWatchedExit unrunRejection?: SubmissionRejectionFact @@ -104,7 +105,7 @@ export function planStructuredAgentSessionDeadGeneration( // Delivery owns failure rows for sends; other startup effects need their own row. // Existing rejection wording is terminal and must remain unchanged. const recordedByDeliveryLoop = - input.exitedDuringStartup.flightOwned || + input.exitedDuringStartup.owned || input.journal.submissions?.().some(isQueuedAgentJournalSubmission) || hasStructuredAgentSessionStartFailureRow(allItems, startKey) if (!recordedByDeliveryLoop) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 87cb788588e0..62602a8e272f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -18,7 +18,10 @@ import { type StructuredAgentSessionGenerationEnded } from './structured-agent-session-child-exit' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' -import { openStructuredAgentSessionProviderChild } from './structured-agent-session-provider-open' +import { + openStructuredAgentSessionProviderChild, + type StructuredAgentSessionFailStartingChild +} from './structured-agent-session-provider-open' export class StructuredAgentSessionEventRecovery { private readonly sinkFailures = new Set() @@ -35,16 +38,7 @@ export class StructuredAgentSessionEventRecovery { now: () => number runtimeState: StructuredAgentSessionHostRuntimeState wakeDelivery: (sessionId: string) => Promise - stopStartingChild: ( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity - ) => void - failStartingChild: ( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity, - error: unknown, - openMs: number - ) => Promise + failStartingChild: StructuredAgentSessionFailStartingChild trackOpen: (open: Promise) => Promise } ) {} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index e1b9dfc3e3da..1f73c74235df 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -32,7 +32,10 @@ export type { StructuredAgentSessionLinkageJournal, StructuredAgentSessionRevisionResolver } from './structured-agent-session-journal-views' -import { StructuredAgentSessionExitObservation } from './structured-agent-session-exit-observation' +import { + StructuredAgentSessionExitObservation, + StructuredAgentSessionRootExitReceipt +} from './structured-agent-session-exit-observation' export type StructuredAgentSessionSinkAdmission = | { accepted: true } @@ -72,6 +75,9 @@ export type StructuredAgentSessionRevisionOptions = StructuredAgentSessionItemAp export type StructuredAgentSessionEventSink = StructuredAgentSessionTransitionSink & { observeExit?: StructuredAgentSessionExitObservation['record'] + /** Adapters report only their acquisition's own spontaneous root exit, never a requested close. */ + observeRootExit?: StructuredAgentSessionRootExitReceipt['observe'] + onRootExit?: StructuredAgentSessionRootExitReceipt['listen'] appendItem( identity: AgentJournalItemIdentity, body: AgentJournalItemBody, @@ -190,6 +196,7 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { onBackpressureChange?: (backpressured: boolean, state: StructuredAgentSessionSinkState) => void }): DeferredStructuredAgentSessionEventSink { const exit = new StructuredAgentSessionExitObservation() + const rootExit = new StructuredAgentSessionRootExitReceipt() const watermarks = { ...DEFAULT_WATERMARKS, ...deps.watermarks } const failed = (error: unknown): void => { deps.logger.error('writing provider events to the chat journal failed', { @@ -260,6 +267,8 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { exitObservation: () => exit, sink: { observeExit: exit.record, + observeRootExit: rootExit.observe, + onRootExit: rootExit.listen, appendItem: (identity, body, options) => void appendItem(identity, body, options), tryAppendItem: appendItem, ...resolvedAppend, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-observation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-observation.ts index 8e1287cadfa1..0e100f393cd3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-observation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-observation.ts @@ -15,3 +15,33 @@ export class StructuredAgentSessionExitObservation { this.close ??= close } } + +/** The first spontaneous root exit the adapter observed for this sink's acquisition, taken where + * the adapter sees it, before output drain, cleanup or host serialization can delay it. The + * listener's own clock times it, so adapter and host clocks never mix. */ +export class StructuredAgentSessionRootExitReceipt { + observed = false + private readonly listeners = new Set<() => void>() + + observe = (): void => { + if (this.observed) { + return + } + this.observed = true + const listeners = [...this.listeners] + this.listeners.clear() + for (const listener of listeners) { + listener() + } + } + + /** Replays an exit already observed; the returned call stops listening. */ + listen = (listener: () => void): (() => void) => { + if (this.observed) { + listener() + return () => {} + } + this.listeners.add(listener) + return () => this.listeners.delete(listener) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.test.ts index 3fe9e3775e62..23f81d9571c2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.test.ts @@ -32,7 +32,7 @@ const ORCA_INTERNAL = /agent_session_|execution_owner|provider_[a-z_]+|[0-9a-f]{8}-[0-9a-f]{4}-|[/\\][\w.-]+[/\\]|Error:|ENOENT/ describe('structuredAgentSessionStartFailure', () => { - it('names a provider request that ran out of time, through wrappers, unless the provider exited', () => { + it('names a provider request that ran out of time, through wrappers and an exit seen around it', () => { const timedOut = withProviderRequestTimeout(new Error('thread/start exceeded 30000ms'), { method: 'thread/start', timeoutMs: 30_000 @@ -56,7 +56,10 @@ describe('structuredAgentSessionStartFailure', () => { ) expect( structuredAgentSessionStartFailure({ error: withObservedProviderExit(wrapped) }).rejection - ).toEqual({ kind: 'providerStartFailed' }) + ).toEqual({ + kind: 'startFailed', + requestTimeout: { method: 'thread/start', timeoutMs: 30_000 } + }) }) it('names a saved Arguments option in start and restart failures without exposing its value', () => { const error = new StructuredAgentArgumentsError( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts index 33e8d6587284..027b51447cf1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-failure-text.ts @@ -149,11 +149,17 @@ export function providerStartupFailureFact(cause?: unknown): SubmissionRejection if (preparation) { return agentSessionFailureFact('launchPreparationFailed', { detail: preparation }) } - const exited = providerExitObserved(cause) + // A request deadline is the most specific explanation: an exit seen around it never replaces it. + const requestTimeout = providerRequestTimeoutOf(cause) + const exited = !requestTimeout && providerExitObserved(cause) return agentSessionFailureFact(exited ? 'providerStartFailed' : 'startFailed', { + // A typed refusal of any reason stays final when only its exit fact reaches the start. + ...(cause instanceof AgentSessionAcquisitionRefusal + ? { refusal: { code: cause.code, details: { reason: cause.reason } } } + : {}), detail: providerDiagnosticOf(cause), argumentProblem: argumentProblemOf(cause), - ...(exited ? {} : { requestTimeout: providerRequestTimeoutOf(cause) }) + requestTimeout }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-held-start.test-support.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-held-start.test-support.ts index 65e0865f2536..855789c0fed3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-held-start.test-support.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-held-start.test-support.ts @@ -1,7 +1,18 @@ -import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import type { + StructuredAgentSessionAdapter, + StructuredAgentSessionLifecycleEvent +} from './structured-agent-session-adapter' +import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import { + providerStartupFailureFact, + structuredAgentSessionStartFailureFact, + withObservedProviderExit +} from './structured-agent-session-failure-text' type Acquire = StructuredAgentSessionAdapter['acquire'] type Open = StructuredAgentSessionAdapter['open'] +type EventHost = { handleAdapterEvent(event: StructuredAgentSessionLifecycleEvent): Promise } export type StructuredAgentSessionHeldStarts = { held(acquire: Acquire): Acquire @@ -10,17 +21,61 @@ export type StructuredAgentSessionHeldStarts = { clear(): void } +const EXIT_KINDS = new Set(['providerExited', 'providerStartFailed', 'startFailed']) + +/** As an adapter reads its own child's exit: a typed refusal the exit carried stays final, and an + * unanswered exit may start once more. */ +export function testAdapterStartupEnding( + failure?: SubmissionRejectionFact +): AgentSessionStartFailure { + const exited = withObservedProviderExit(new Error('provider exited before open answered')) + const final = failure && (failure.refusal !== undefined || !EXIT_KINDS.has(failure.kind)) + return new AgentSessionStartFailure( + exited, + final ? 'never' : 'safe', + failure + ? structuredAgentSessionStartFailureFact({ exit: failure }) + : providerStartupFailureFact(exited) + ) +} + +/** Opens wait until proved; as a real adapter, a child's own exit answers its waiting open. */ export function structuredAgentSessionHeldStarts( - _host: () => unknown + host: () => EventHost | undefined ): StructuredAgentSessionHeldStarts { const held = new Map void>() const waiting = new Map>() + const endings = new Map() + const ended = new Map void>() + const observed = new WeakSet() + const observe = (): void => { + const target = host() + if (!target || observed.has(target)) { + return + } + observed.add(target) + const handle = target.handleAdapterEvent.bind(target) + target.handleAdapterEvent = async (event) => { + const ending = + event.type === 'ended' && event.cause === 'unexpected-exit' + ? testAdapterStartupEnding(event.failure) + : undefined + if (ending && event.type === 'ended') { + endings.set(event.acquisitionGeneration, ending) + } + await handle(event) + if (ending && event.type === 'ended') { + ended.get(event.acquisitionGeneration)?.(ending) + } + } + } const prove: StructuredAgentSessionHeldStarts['prove'] = async (child) => { held.get(child.acquisitionGeneration)?.() await Promise.resolve() } return { held: (acquire) => async (input) => { + observe() const acquired = await acquire(input) const generation = acquired.acquisitionGeneration waiting.set(generation, new Promise((resolve) => held.set(generation, resolve))) @@ -28,15 +83,28 @@ export function structuredAgentSessionHeldStarts( }, wrap: (adapter) => { const open: Open = async (input) => { + observe() const generation = input.acquisitionGeneration - if (generation) { - await waiting.get(generation) + const ending = endings.get(generation) + if (ending) { + throw ending + } + const exit = new Promise((_resolve, reject) => ended.set(generation, reject)) + void exit.catch(() => {}) + try { + await Promise.race([waiting.get(generation), exit]) waiting.delete(generation) held.delete(generation) + return await Promise.race([adapter.open(input), exit]) + } finally { + ended.delete(generation) } - return adapter.open(input) } - return { ...adapter, open } + const startupEnding: NonNullable = ( + _sessionId, + generation + ) => endings.get(generation) + return { ...adapter, open, startupEnding } }, prove, clear: () => { @@ -45,6 +113,7 @@ export function structuredAgentSessionHeldStarts( } held.clear() waiting.clear() + endings.clear() } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts index 01e96ab8862c..477b947aeac2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts @@ -21,7 +21,10 @@ import { StructuredAgentSessionStartupAttempts, type StructuredAgentSessionExpiredStartup } from './structured-agent-session-startup-attempt' -import { StructuredAgentSessionStartupExpiredError } from './structured-agent-session-startup-attempt-contract' +import { + StructuredAgentSessionStartupContractError, + StructuredAgentSessionStartupExpiredError +} from './structured-agent-session-startup-attempt-contract' export class StructuredAgentSessionHostRuntimeState { private readonly eventSinks = new Map() @@ -50,6 +53,16 @@ export class StructuredAgentSessionHostRuntimeState { if (expired.child !== null) { onStartupExpired?.(expired) } + }, + // The exited child still ends through its own exit; only the start's waits are cancelled. + breach: ({ sessionId, attemptId, agent }) => { + deps.logger.warn('an agent adapter gave no startup outcome after its process exited', { + scope: 'startup-contract', + sessionId, + attemptId, + agent + }) + this.acquireAborts.abort(sessionId, new StructuredAgentSessionStartupContractError(agent)) } }) this.leaseRenewer = new StructuredAgentSessionLeaseRenewer({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 10a86a5748ea..40f2c7742b24 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -116,7 +116,6 @@ export class StructuredAgentSessionHost { now: () => this.now(), runtimeState: this.runtimeState, wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), - stopStartingChild: (...args) => this.lifetime.stopStartingChild(...args), failStartingChild: (...args) => this.lifetime.failStartingChild(...args), trackOpen: (open) => this.tasks.trackAttach(open) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts index 26e66bfacb0f..281add0eea6b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts @@ -12,6 +12,12 @@ import { hostTestState } from './structured-agent-session-host-test-harness' import { HOST_TEST_SESSION, hostTestMessage } from './structured-agent-session-host-test-data' +import canonicalRefusal from '../../codex/app-server/fixtures/start-resume-canonical-filename-refusal.json' +import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' +import { acpStartFailure } from '../../acp/acp-startup-outcome' +import { agentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' +import { structuredAgentSessionStartFailureRowIdentity } from '../../../shared/agent-session/failures/structured-agent-session-start-failure-row-key' const OPENED: StructuredAgentSessionProviderOpenResult = { reportedOptions: { model: 'gpt-test' } @@ -78,3 +84,69 @@ it('journals a typed writer refusal and cleans up the real starting child', asyn expect(close).toHaveBeenCalledOnce() expect(dispatch).not.toHaveBeenCalled() }) + +it("records one standalone row in the start's words when a direct attach fails with nothing queued", async () => { + const { host, open, close, dispatch, pending } = awaitOpen() + await attach() + await vi.waitFor(() => expect(open).toHaveBeenCalledOnce()) + pending.reject( + codexStartFailure( + new CodexAppServerRequestError( + 'thread/resume', + canonicalRefusal.code, + canonicalRefusal.message + ) + ) + ) + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()) + const snapshot = await host.journalSnapshot(HOST_TEST_SESSION) + expect(snapshot.items.filter((item) => item.body.kind === 'status')).toEqual([ + expect.objectContaining({ + itemId: agentJournalItemKey(structuredAgentSessionStartFailureRowIdentity('awaited-open')), + body: expect.objectContaining({ + tone: 'error', + failure: expect.objectContaining({ kind: 'startFailed' }) + }) + }) + ]) + expect(snapshot.submissions).toEqual([]) + expect(dispatch).not.toHaveBeenCalled() +}) + +it("keeps the exit's words for a direct attach whose exit settled before its final answer", async () => { + const { host, open, dispatch, pending } = awaitOpen() + await attach() + await vi.waitFor(() => expect(open).toHaveBeenCalledOnce()) + const child = host.collaboratorsForTests().sessions.get(HOST_TEST_SESSION)!.child! + await host.handleAdapterEvent({ + type: 'ended', + sessionId: HOST_TEST_SESSION, + fence: child.fence, + acquisitionGeneration: child.generation!, + cause: 'unexpected-exit', + reason: 'opencode exited', + failure: { kind: 'providerExited' } + }) + // The agent's sign-in answer drained after its exit; it stays final but words nothing new. + pending.reject( + acpStartFailure( + new AgentSessionAcquisitionRefusal('OpenCode requires sign-in', 'notSignedIn'), + true + ) + ) + await vi.waitFor(async () => { + const rows = (await host.journalSnapshot(HOST_TEST_SESSION)).items.filter( + (item) => item.body.kind === 'status' + ) + expect(rows).toEqual([ + expect.objectContaining({ + itemId: agentJournalItemKey(structuredAgentSessionStartFailureRowIdentity('awaited-open')), + body: expect.objectContaining({ + failure: expect.objectContaining({ kind: 'providerStartFailed' }) + }) + }) + ]) + }) + await vi.waitFor(() => expect(open).toHaveBeenCalledOnce()) + expect(dispatch).not.toHaveBeenCalled() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts index 021844e711fe..3e9645f94fc3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.ts @@ -1,70 +1,124 @@ -import { sameProviderChild } from './structured-agent-session-provider-child' import { waitForPromiseWithSignal } from '../../../shared/abort-signal-reason' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' import { settleStructuredAgentSessionProviderStarted, type StructuredAgentSessionProviderStartedContext } from './structured-agent-session-provider-started' +import { + AgentSessionStartFailure, + agentSessionStartFailure +} from './structured-agent-session-start-failure' +import { + isStructuredAgentSessionStartupContractBreach, + isStructuredAgentSessionStartupExpired +} from './structured-agent-session-startup-attempt-contract' + +export type StructuredAgentSessionFailStartingChild = ( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + failure: AgentSessionStartFailure, + openMs: number +) => Promise -/** Await protocol construction outside conversation serialization. */ +/** The one owner of a starting child's outcome: it awaits the adapter's open and the start's + * required settlement outside serialization, then gives the start its verdict once. */ export async function openStructuredAgentSessionProviderChild( context: StructuredAgentSessionProviderStartedContext & { - failStartingChild: ( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity, - error: unknown, - openMs: number - ) => Promise + failStartingChild: StructuredAgentSessionFailStartingChild }, sessionId: string, child: StructuredAgentSessionProviderChildIdentity ): Promise { - if (!context.runtimeState.startupAttempts.onClock(sessionId, child)) { + const { startupAttempts, acquireAborts } = context.runtimeState + // Once this owner is done, nothing reads how the acquisition ended before ready. + const retire = (): void => { + if (child.generation !== null) { + context.deps.adapter.retireStartup?.(sessionId, child.generation) + } + } + if (!startupAttempts.own(sessionId, child)) { + retire() return } - const attempt = context.runtimeState.acquireAborts.begin(sessionId, false, false, () => { - const current = context.sessions.get(sessionId)?.child - return current !== undefined && current !== null && sameProviderChild(current, child) - }) + const attempt = acquireAborts.begin(sessionId) const openedAt = context.now() try { - attempt.signal.throwIfAborted() - const acquisitionGeneration = child.generation - if (acquisitionGeneration === null) { - throw new Error('An unopened provider child has no acquisition generation') - } - const result = await waitForPromiseWithSignal( - context.deps.adapter.open({ - sessionId, - fence: child.fence, - acquisitionGeneration, - signal: attempt.signal - }), - attempt.signal + const outcome = await openAndSettle(context, sessionId, child, attempt.signal).catch( + (error: unknown) => + attempt.signal.aborted + ? stoppedStart(attempt.signal.reason) + : error instanceof Error + ? error + : new Error(String(error)) ) - if (attempt.signal.aborted) { + if (outcome === 'ready') { return } - await settleStructuredAgentSessionProviderStarted( - context, - { ...result, sessionId, fence: child.fence, acquisitionGeneration }, - attempt.signal - ) - } catch (error) { - // Aborted startup already has a Stop/close/quit/expiry owner. - if (!attempt.signal.aborted) { - const openMs = Math.max(0, context.now() - openedAt) - await context - .failStartingChild(sessionId, child, error, openMs) - .catch((closeError: unknown) => - context.deps.logger.warn('closing a failed provider open did not finish', { - scope: 'provider-open-close', - sessionId, - error: closeError - }) - ) + if (!(outcome instanceof Error)) { + startupAttempts.released(sessionId, child) + return } + const failure = agentSessionStartFailure(outcome) + // Queued first so the start's row precedes any send that wakes on its verdict. + const cleanup = context.failStartingChild( + sessionId, + child, + failure, + Math.max(0, context.now() - openedAt) + ) + startupAttempts.failed(sessionId, child, failure) + await cleanup.catch((error: unknown) => + context.deps.logger.warn('closing a failed provider open did not finish', { + scope: 'provider-open-close', + sessionId, + error + }) + ) } finally { + startupAttempts.disown(sessionId, child) attempt.end() + retire() + } +} + +async function openAndSettle( + context: StructuredAgentSessionProviderStartedContext, + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + signal: AbortSignal +): Promise<'ready' | 'stale' | Error> { + signal.throwIfAborted() + const acquisitionGeneration = child.generation + if (acquisitionGeneration === null) { + throw new Error('An unopened provider child has no acquisition generation') + } + const result = await waitForPromiseWithSignal( + context.deps.adapter.open({ sessionId, fence: child.fence, acquisitionGeneration, signal }), + signal + ) + return waitForPromiseWithSignal( + settleStructuredAgentSessionProviderStarted( + context, + { ...result, sessionId, fence: child.fence, acquisitionGeneration }, + signal + ), + signal + ) +} + +/** Stop, close and quit settle through their own owner. The deadline and the contract guard + * already decided the start; the owner still records its row. */ +function stoppedStart(reason: unknown): 'stale' | AgentSessionStartFailure { + if (isStructuredAgentSessionStartupExpired(reason)) { + return new AgentSessionStartFailure(reason, 'never', agentSessionFailureFact('hostStopped')) + } + if (isStructuredAgentSessionStartupContractBreach(reason)) { + return new AgentSessionStartFailure( + reason, + 'never', + agentSessionFailureFact('providerStartFailed') + ) } + return 'stale' } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started-outcome.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started-outcome.test.ts new file mode 100644 index 000000000000..98125b55a428 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started-outcome.test.ts @@ -0,0 +1,81 @@ +import { beforeEach, expect, it, vi } from 'vitest' +import canonicalRefusal from '../../codex/app-server/fixtures/start-resume-canonical-filename-refusal.json' +import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' +import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' +import { readStructuredRewindRecovery } from './structured-rewind-recovery' +import type * as RewindRecovery from './structured-rewind-recovery' +import { + settleStructuredAgentSessionProviderStarted, + type StructuredAgentSessionProviderStartedContext +} from './structured-agent-session-provider-started' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' + +vi.mock('./structured-rewind-recovery', async (importOriginal) => ({ + ...(await importOriginal()), + readStructuredRewindRecovery: vi.fn() +})) + +const SESSION = 'session-1' +const OPENED = { sessionId: SESSION, fence: 3, acquisitionGeneration: 'g-1', reportedOptions: {} } +const died = codexStartFailure(new CodexAppServerExitError('codex app-server connection ended')) +let ending: AgentSessionStartFailure | undefined + +function context(childGeneration = 'g-1'): StructuredAgentSessionProviderStartedContext { + const session = { + journal: {}, + child: { generation: childGeneration, fence: 3, phase: 'starting' } + } + const members = { + deps: { + adapter: { startupEnding: () => ending }, + store: { getRecord: () => null, getConversation: () => ({ currentFence: 3 }) }, + logger: { warn: vi.fn() } + }, + sessions: new Map([[SESSION, session]]), + serialize: (_sessionId: string, task: () => Promise) => task(), + now: () => 0, + runtimeState: { + startupAttempts: { onClock: () => true, rootExit: () => new Promise(() => {}) }, + acquireAborts: {} + }, + wakeDelivery: vi.fn() + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: settlement reads only these members before its verdict. + return members as unknown as StructuredAgentSessionProviderStartedContext +} + +beforeEach(() => { + ending = undefined + vi.mocked(readStructuredRewindRecovery).mockReset().mockResolvedValue(null) +}) + +it("keeps a required read's final answer over the child's later death", async () => { + const answer = new CodexAppServerRequestError( + 'thread/read', + canonicalRefusal.code, + canonicalRefusal.message + ) + vi.mocked(readStructuredRewindRecovery).mockRejectedValue(codexStartFailure(answer)) + ending = died + const outcome = await settleStructuredAgentSessionProviderStarted(context(), OPENED) + expect(outcome).toMatchObject({ retry: 'never', cause: answer }) +}) + +it('answers an unanswered death after a successful open with the adapter verdict', async () => { + ending = died + expect(await settleStructuredAgentSessionProviderStarted(context(), OPENED)).toBe(died) + expect(died.retry).toBe('safe') +}) + +it('retires only its own owner once the child was replaced', async () => { + expect(await settleStructuredAgentSessionProviderStarted(context('g-2'), OPENED)).toBe('stale') +}) + +it('ends the start on a reported death while its bookkeeping is still blocked', async () => { + ending = died + const blocked = context() + blocked.flushLifecycle = () => new Promise(() => {}) + blocked.runtimeState.startupAttempts.rootExit = () => Promise.resolve() + expect(await settleStructuredAgentSessionProviderStarted(blocked, OPENED)).toBe(died) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts index a0011bfd5046..272f8b894fab 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts @@ -1,14 +1,11 @@ import type { StructuredAgentSessionOptionsReportedEvent } from './structured-agent-session-adapter' -import type { StructuredAgentSessionProviderOpenResult } from './structured-agent-session-provider-open-contract' import type { StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession, - StructuredAgentSessionProviderChildIdentity + StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { markProviderChildStarted, - sameProviderChild, - structuredAgentSessionConversationFence + sameProviderChild } from './structured-agent-session-provider-child' import { saveStructuredAgentSessionListing, @@ -20,14 +17,16 @@ import { noteStructuredAgentSessionReportedOptions } from './structured-agent-session-ran-options' import type { StructuredAgentSessionAcquireAborts } from './structured-agent-session-acquire-aborts' -import { recordAgentSessionProviderHandle } from '../../runtime/agent-session/ownership/agent-session-provider-handle-transition' -import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' import { settleUnheldStructuredAgentSessionTurns } from './structured-agent-session-unheld-turns' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' import { - applyStructuredRewindRecovery, - readStructuredRewindRecovery, - type StructuredRewindRecoveryEvidence -} from './structured-rewind-recovery' + applyRewindAtStart, + readRewindAtStart, + recordStartedHandle, + refusedStart, + type OpenedProvider +} from './structured-agent-session-start-provider-answer' +import type { StructuredRewindRecoveryEvidence } from './structured-rewind-recovery' export type StructuredAgentSessionProviderStartedContext = { deps: StructuredAgentSessionHostDeps @@ -36,214 +35,152 @@ export type StructuredAgentSessionProviderStartedContext = { now: () => number publishStatus?: (sessionId: string) => void runtimeState: { - startupAttempts: Pick + startupAttempts: Pick< + StructuredAgentSessionStartupAttempts, + 'ready' | 'onClock' | 'readiness' | 'own' | 'disown' | 'failed' | 'released' | 'rootExit' + > acquireAborts: Pick } /** Commits the child's buffered lifecycle events. */ flushLifecycle?: (sessionId: string) => Promise /** Release startupโ€™s delivery barrier and drain all currently handable sends. */ wakeDelivery: (sessionId: string) => Promise - stopStartingChild: (sessionId: string, child: StructuredAgentSessionProviderChildIdentity) => void } -type OpenedProvider = StructuredAgentSessionProviderOpenResult & { - sessionId: string - fence: number - acquisitionGeneration: string -} +/** A failure is the adapter's verdict on required provider work or on this child ending before it + * was ready, or Orca refusing what the open answered; `stale` retires only this owner. */ +export type StructuredAgentSessionProviderStartedOutcome = + | 'ready' + | 'stale' + | AgentSessionStartFailure export async function settleStructuredAgentSessionProviderStarted( context: StructuredAgentSessionProviderStartedContext, event: OpenedProvider, signal?: AbortSignal -): Promise { +): Promise { // Read provider state outside serialization so sends can still be accepted. const rewind = await readRewindAtStart(context, event, signal) - // Restored history is committed and the child's own events drained before its turns are judged. - await context.flushLifecycle?.(event.sessionId) - // Follow attach publication so its proven lease has committed. - await context.serialize(event.sessionId, async () => { - signal?.throwIfAborted() - const session = context.sessions.get(event.sessionId) - const child = { generation: event.acquisitionGeneration, fence: event.fence } - // Stale or expired startup cannot lift the current childโ€™s barrier. - if ( - !session?.child || - session.child.phase !== 'starting' || - !sameProviderChild(session.child, child) || - !context.runtimeState.startupAttempts.onClock(event.sessionId, child) - ) { - return null - } - if (!(await recordStartedHandle(context, event))) { - context.stopStartingChild(event.sessionId, child) - return null - } - if (!(await applyRewindAtStart(context, event.sessionId, session, rewind))) { - context.stopStartingChild(event.sessionId, child) - return null - } - if ( - !context.runtimeState.startupAttempts.onClock(event.sessionId, child) || - !markProviderChildStarted(session, child) - ) { - return null - } - await settleUnheldStructuredAgentSessionTurns( - session.journal, - event.sessionId, - child.fence, - event.acquisitionGeneration, - context.deps.adapter.liveTurnId?.(event.sessionId) - ).catch((error: unknown) => - context.deps.logger.warn('settling turns the agent is not running failed', { - scope: 'unheld-turns', - sessionId: event.sessionId, - error - }) - ) - const sourceChild = session.child - context.runtimeState.startupAttempts.ready(event.sessionId, child) - noteStructuredAgentSessionRuntimePermission(session.child, event.reportedOptions, true) - context.publishStatus?.(event.sessionId) - await noteStructuredAgentSessionReportedOptions( - context, - event.sessionId, - child.fence, - event.reportedOptions - ) - void context.wakeDelivery(event.sessionId) - noteStructuredAgentSessionProviderStarted( - context.deps, - event.sessionId, - () => - context.sessions.get(event.sessionId)?.child === sourceChild && - sourceChild?.catalogSource?.() !== false - ) - if (event.catalogListing) { - saveStructuredAgentSessionListing( - context, - event.sessionId, - event.catalogListing, - () => - context.sessions.get(event.sessionId)?.child === sourceChild && - sourceChild?.catalogSource?.() !== false - ) + // Required provider work's own verdict outranks this child's later death. + if (rewind instanceof AgentSessionStartFailure) { + return rewind + } + const child = { generation: event.acquisitionGeneration, fence: event.fence } + // Only the adapter's word on this exact acquisition says it died; absence alone says nothing. + const ended = () => + context.deps.adapter.startupEnding?.(event.sessionId, event.acquisitionGeneration) + const local: LocalSettlement = { started: false, fenced: false } + // A death the adapter reports ends the start without waiting on its bookkeeping below, but + // never once Orca's own handle or rewind write began: only that write's answer says whether + // the start may run again, and the contract guard bounds the wait for it. + const died = context.runtimeState.startupAttempts.rootExit(event.sessionId, child).then(() => { + const ending = local.started ? undefined : ended() + if (!ending) { + return new Promise(() => {}) } - context.publishStatus?.(event.sessionId) - return undefined + local.fenced = true + return ending }) + return Promise.race([died, settleReady(context, event, rewind, ended, local, signal)]) } -/** Reconcile uncertain rewind before delivery; persistent uncertainty fails startup. */ -async function readRewindAtStart( - context: StructuredAgentSessionProviderStartedContext, - event: OpenedProvider, - signal?: AbortSignal -): Promise { - const { sessionId } = event - const journal = context.sessions.get(sessionId)?.journal - if (!journal) { - return null - } - try { - return await readStructuredRewindRecovery( - context.deps, - sessionId, - journal, - structuredAgentSessionConversationFence(context.deps.store, sessionId), - context.deps.adapter, - signal, - recordWithStartedHandle(context, event) - ) - } catch (error) { - return rewindRecoveryFailed(context, sessionId, error, signal) - } -} - -/** Predict the committed handle for rewind cuts; if recording is refused, retain the existing record. */ -function recordWithStartedHandle( - context: StructuredAgentSessionProviderStartedContext, - { sessionId, fence, link }: OpenedProvider -): AgentSessionRecord | null { - const record = context.deps.store.getRecord(sessionId) - try { - return record && link - ? recordAgentSessionProviderHandle({ record, fence, link, now: context.now() }) - : record - } catch { - return record - } -} - -async function applyRewindAtStart( - context: StructuredAgentSessionProviderStartedContext, - sessionId: string, - session: StructuredAgentSessionHostSession, - evidence: StructuredRewindRecoveryEvidence | 'failed' -): Promise { - if (evidence === 'failed') { - return false - } - try { - await applyStructuredRewindRecovery( - context.deps, - sessionId, - session.journal, - structuredAgentSessionConversationFence(context.deps.store, sessionId), - evidence - ) - return true - } catch (error) { - rewindRecoveryFailed(context, sessionId, error) - return false - } -} +/** `fenced`: the death decided the start, so its local writes never begin. */ +type LocalSettlement = { started: boolean; fenced: boolean } -function rewindRecoveryFailed( +function settleReady( context: StructuredAgentSessionProviderStartedContext, - sessionId: string, - error: unknown, + event: OpenedProvider, + rewind: StructuredRewindRecoveryEvidence, + ended: () => AgentSessionStartFailure | undefined, + local: LocalSettlement, signal?: AbortSignal -): 'failed' { - if (!signal?.aborted) { - context.deps.logger.warn('settling a rewind in doubt at the start failed', { - scope: 'rewind-recovery', - sessionId, - error - }) - } - return 'failed' -} - -/** Persist the opened handle before delivery so future starts can resume; false if no handle is proven. */ -async function recordStartedHandle( - context: StructuredAgentSessionProviderStartedContext, - event: OpenedProvider -): Promise { - const { store, logger } = context.deps - const { link } = event - if (!link) { - // Only an already-proven handle permits open without a new link. - return store.getRecord(event.sessionId)?.lease.provenHandleLinkId != null - } - try { - const committed = await store.transitionHandoff(event.sessionId, (record) => - recordAgentSessionProviderHandle({ record, fence: event.fence, link, now: context.now() }) - ) - // Inherit only a committed new context; a resumed CLI keeps its own later title. - if (event.newConversation ?? (link.origin === 'created' || link.origin === 'forked')) { - context.deps.nameProviderConversation?.(committed) - } - return true - } catch (error) { - logger.warn('recording the handle a started provider answered with failed', { - scope: 'provider-started-handle', - sessionId: event.sessionId, - error +): Promise { + const child = { generation: event.acquisitionGeneration, fence: event.fence } + // Restored history is committed and the child's own events drained before its turns are judged. + return Promise.resolve(context.flushLifecycle?.(event.sessionId)).then(() => + // Follow attach publication so its proven lease has committed. + context.serialize(event.sessionId, async () => { + signal?.throwIfAborted() + const session = context.sessions.get(event.sessionId) + const ending = ended() + const current = session?.child && sameProviderChild(session.child, child) ? session : null + if (!current?.child) { + return ending ?? 'stale' + } + // Stale or expired startup cannot lift the current childโ€™s barrier. + if ( + current.child.phase !== 'starting' || + !context.runtimeState.startupAttempts.onClock(event.sessionId, child) + ) { + return 'stale' + } + if (ending || local.fenced) { + return ending ?? 'stale' + } + local.started = true + if ( + !(await recordStartedHandle(context, event)) || + !(await applyRewindAtStart(context, event.sessionId, current, rewind)) + ) { + return refusedStart() + } + signal?.throwIfAborted() + // The writes succeeded; a death meanwhile is still this start's outcome. + const endedWhileWriting = ended() + if (endedWhileWriting) { + return endedWhileWriting + } + if ( + !context.runtimeState.startupAttempts.onClock(event.sessionId, child) || + !markProviderChildStarted(current, child) + ) { + return 'stale' + } + await settleUnheldStructuredAgentSessionTurns( + current.journal, + event.sessionId, + child.fence, + event.acquisitionGeneration, + context.deps.adapter.liveTurnId?.(event.sessionId) + ).catch((error: unknown) => + context.deps.logger.warn('settling turns the agent is not running failed', { + scope: 'unheld-turns', + sessionId: event.sessionId, + error + }) + ) + const sourceChild = current.child + context.runtimeState.startupAttempts.ready(event.sessionId, child) + noteStructuredAgentSessionRuntimePermission(current.child, event.reportedOptions, true) + context.publishStatus?.(event.sessionId) + await noteStructuredAgentSessionReportedOptions( + context, + event.sessionId, + child.fence, + event.reportedOptions + ) + void context.wakeDelivery(event.sessionId) + noteStructuredAgentSessionProviderStarted( + context.deps, + event.sessionId, + () => + context.sessions.get(event.sessionId)?.child === sourceChild && + sourceChild?.catalogSource?.() !== false + ) + if (event.catalogListing) { + saveStructuredAgentSessionListing( + context, + event.sessionId, + event.catalogListing, + () => + context.sessions.get(event.sessionId)?.child === sourceChild && + sourceChild?.catalogSource?.() !== false + ) + } + context.publishStatus?.(event.sessionId) + return 'ready' as const }) - return false - } + ) } /** Prompt catalog recheck after startup may prove installation or restored sign-in; the probe decides. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts index 34d3d4117f1c..083758d6487d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts @@ -355,7 +355,8 @@ describe('host rewind', () => { expect( await attachAndStart(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) ).toMatchObject({ ok: true }) - expect(startedChild()).toBeNull() + // The start's owner stops the child once its required rewind read failed. + await vi.waitFor(() => expect(startedChild()).toBeNull()) expect(store.getRecord(HOST_TEST_SESSION)?.rewind?.phase).toBe('prepared') expect( await attachAndStart(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts index a75997ad6f45..cb74e7ba38f2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-attempts.ts @@ -9,18 +9,22 @@ import { agentSessionFailureFact } from '../../../shared/agent-session/failures/ import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionAttachLock } from './structured-agent-session-attach-lock' import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-agent-start' -import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import type { + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity +} from './structured-agent-session-host-types' import { structuredAgentSessionStartFlightLock } from './structured-agent-session-start-flight' -import { isStructuredAgentSessionStartupExpired } from './structured-agent-session-startup-attempt-contract' +import { + isStructuredAgentSessionStartupContractBreach, + isStructuredAgentSessionStartupExpired +} from './structured-agent-session-startup-attempt-contract' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' import { refusedStructuredAgentSessionStart } from './structured-agent-session-start-eligibility' import { refusedStructuredAgentSessionStart as failedStart, recordStructuredAgentSessionStartFailure } from './structured-agent-session-start-failure-row' -import { - structuredAgentSessionStartFailure, - structuredAgentSessionStartFailureFact -} from './structured-agent-session-failure-text' +import { structuredAgentSessionStartFailure } from './structured-agent-session-failure-text' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' import { sameProviderChild } from './structured-agent-session-provider-child' @@ -47,138 +51,146 @@ export function registerStructuredAgentSessionStart( signal: acquire.signal, children }) - const outcome = context.tasks.trackAttach( - Promise.resolve() - .then(async (): Promise => { - let result: StructuredAgentSessionResumeOutcome = failedStructuredAgentSessionStart() - for (let number = 0; number < 5; number++) { + const attempting = Promise.resolve() + .then(async (): Promise => { + let result: StructuredAgentSessionResumeOutcome = failedStructuredAgentSessionStart() + for (let number = 0; number < 5; number++) { + throwIfSignalAborted(acquire.signal) + if (attempts.pastCeiling(registeredAt)) { + result = { ...failedStructuredAgentSessionStart(), expired: true } + break + } + const lock = structuredAgentSessionStartFlightLock({ + sessionId, + attemptId, + signal: acquire.signal, + flights, + serialize: context.serialize + }) + const stillQueued = await lock.commit(async () => + structuredAgentSessionStartStillQueued(context.sessions.get(sessionId), startedFor) + ) + if (!stillQueued) { + return { ...failedStructuredAgentSessionStart(), aborted: true } + } + result = await run(lock, attemptId) + if (result.ok) { + // The child's open owns its outcome; readiness ends once that open gave it. + const ready = await lock.wait(handle.ended) throwIfSignalAborted(acquire.signal) - if (attempts.pastCeiling(registeredAt)) { - result = { ...failedStructuredAgentSessionStart(), expired: true } - break + if (ready === 'ready') { + return result } - const lock = structuredAgentSessionStartFlightLock({ - sessionId, - attemptId, - signal: acquire.signal, - flights, - serialize: context.serialize - }) - const stillQueued = await lock.commit(async () => - structuredAgentSessionStartStillQueued(context.sessions.get(sessionId), startedFor) + const owned = handle.result + result = await lock.commit(async () => + structuredAgentSessionOpenFailure(context.sessions.get(sessionId), owned) ) - if (!stillQueued) { - return { ...failedStructuredAgentSessionStart(), aborted: true } - } - result = await run(lock, attemptId) - if (result.ok) { - const ready = await lock.wait(handle.ended) - throwIfSignalAborted(acquire.signal) - if (ready === 'ready') { - return result - } - result = await lock.commit(async () => { - const session = context.sessions.get(sessionId) - const ended = session?.lastEndedChild - const child = handle.result.child - if ( - !child || - !ended || - !sameProviderChild(ended, child) || - !ended.rootGone || - (session?.child && sameProviderChild(session.child, child)) - ) { - return refusedStructuredAgentSessionStart( - 'agent_session_ownership_unknown', - { reason: 'ownerUnproven' }, - 'The previous agent process could not be verified as stopped.' - ) - } - const failure = structuredAgentSessionStartFailureFact({ - exit: handle.result.failure ?? ended.failure - }) - return { - ...failedStructuredAgentSessionStart(), - startKey: ended.generation, - failure, - retry: - handle.result.retry ?? - (ended.cause === 'exit' && failure.kind === 'providerStartFailed') - } - }) - } - if (result.ok || result.aborted || result.expired || !result.retry || number === 4) { - break - } - attempts.abandon(sessionId, attemptId) - await waitForPromiseWithSignal(delay(100 * 2 ** number), acquire.signal) - throwIfSignalAborted(acquire.signal) - attemptId = randomUUID() - handle = attempts.begin(sessionId, attemptId, registeredAt) - register() } - return result - }) - .catch((error: unknown): StructuredAgentSessionResumeOutcome => { - if (!acquire.signal.aborted) { - context.deps.logger.warn('starting the agent failed', { - scope: 'agent-start', - sessionId, - error - }) - return { - ...failedStructuredAgentSessionStart(), - failure: agentSessionFailureFact('hostFault') - } + if (result.ok || result.aborted || result.expired || !result.retry || number === 4) { + break } - return failedStructuredAgentSessionStart() - }) - .then(async (result): Promise => { - if (result.ok) { - return result - } - return context.serialize(sessionId, async () => { - const settled = acquire.signal.aborted - ? { - ...failedStructuredAgentSessionStart(), - ...(isStructuredAgentSessionStartupExpired(acquire.signal.reason) - ? { expired: true as const } - : { aborted: true as const }) - } - : result - if (!settled.aborted && (startedFor !== undefined || children.length > 0)) { - const session = context.sessions.get(sessionId) - if (session) { - const record = context.deps.store.getRecord(sessionId) - const failure = failedStart(record, settled) - await recordStructuredAgentSessionStartFailure( - { journal: session.journal, fence: record?.lease.runtimeFence ?? 0 }, - { - startKey: - children.at(-1)?.generation ?? failure.startKey ?? startedFor ?? attemptId, - ...structuredAgentSessionStartFailure( - failure.cause, - structuredAgentSessionFailureWordsContext(record) - ) - }, - startedFor ?? null - ) - } - } - return settled - }) - }) - .finally(() => { - acquire.end() - flights.end(sessionId, attemptId) attempts.abandon(sessionId, attemptId) - context.publishStatus(sessionId) - }) - ) + await waitForPromiseWithSignal(delay(100 * 2 ** number), acquire.signal) + throwIfSignalAborted(acquire.signal) + attemptId = randomUUID() + handle = attempts.begin(sessionId, attemptId, registeredAt) + register() + } + return result + }) + .catch((error: unknown): StructuredAgentSessionResumeOutcome => { + if (!acquire.signal.aborted) { + context.deps.logger.warn('starting the agent failed', { + scope: 'agent-start', + sessionId, + error + }) + return { + ...failedStructuredAgentSessionStart(), + failure: agentSessionFailureFact('hostFault') + } + } + return failedStructuredAgentSessionStart() + }) + let recorded: Promise = Promise.resolve() + // Stop, close, quit, the deadline and the contract guard decide the start at once, even while + // its attempt or the session's serialized work is still held; late attempt work fails the + // flight's commit check. The failure row is queued before callers wake, so their work follows + // it, but they never wait for it to be written. + const outcome = Promise.race([ + attempting, + stoppedStructuredAgentSessionStartSignal(acquire.signal) + ]) + .then((result): StructuredAgentSessionResumeOutcome => { + const settled = + result.ok || !acquire.signal.aborted + ? result + : stoppedStructuredAgentSessionStart(acquire.signal.reason) + if (!settled.ok && !settled.aborted && (startedFor !== undefined || children.length > 0)) { + const key = { generation: children.at(-1)?.generation, startedFor, attemptId } + recorded = context + .serialize(sessionId, () => + recordStructuredAgentSessionFlightFailure(context, sessionId, settled, key) + ) + .catch((error: unknown) => + context.deps.logger.warn('recording a failed start did not finish', { + scope: 'agent-start', + sessionId, + error + }) + ) + } + return settled + }) + .finally(() => { + acquire.end() + flights.end(sessionId, attemptId) + attempts.abandon(sessionId, attemptId) + context.publishStatus(sessionId) + }) + context.tasks.trackAttach(outcome.then(() => Promise.all([attempting, recorded]))) register() return outcome } +function stoppedStructuredAgentSessionStartSignal( + signal: AbortSignal +): Promise { + return new Promise((resolve) => { + const stopped = (): void => resolve(stoppedStructuredAgentSessionStart(signal.reason)) + if (signal.aborted) { + stopped() + } else { + signal.addEventListener('abort', stopped, { once: true }) + } + }) +} + +async function recordStructuredAgentSessionFlightFailure( + context: StructuredAgentSessionAttachContext, + sessionId: string, + settled: Extract, + key: { generation: string | null | undefined; startedFor: string | undefined; attemptId: string } +): Promise { + const session = context.sessions.get(sessionId) + if (!session) { + return + } + const record = context.deps.store.getRecord(sessionId) + const failure = failedStart(record, settled) + await recordStructuredAgentSessionStartFailure( + { journal: session.journal, fence: record?.lease.runtimeFence ?? 0 }, + { + startKey: key.generation ?? failure.startKey ?? key.startedFor ?? key.attemptId, + ...structuredAgentSessionStartFailure( + failure.cause, + structuredAgentSessionFailureWordsContext(record) + ) + }, + key.startedFor ?? null + ) +} + export function failedStructuredAgentSessionStart(): Extract< StructuredAgentSessionResumeOutcome, { ok: false } @@ -208,3 +220,52 @@ export function structuredAgentSessionStartStillQueued( ) === true ) } + +/** A safe verdict replays only once this child's exit is proven; no verdict replays nothing. */ +function structuredAgentSessionOpenFailure( + session: Pick | undefined, + { + child, + failure + }: { + child: StructuredAgentSessionProviderChildIdentity | null + failure?: AgentSessionStartFailure + } +): StructuredAgentSessionResumeOutcome { + const ended = session?.lastEndedChild + const exitProven = + !!child && + !!ended?.rootGone && + sameProviderChild(ended, child) && + !(session?.child && sameProviderChild(session.child, child)) + if (!child || !failure || (failure.retry === 'safe' && !exitProven)) { + return refusedStructuredAgentSessionStart( + 'agent_session_ownership_unknown', + { reason: 'ownerUnproven' }, + 'The previous agent process could not be verified as stopped.' + ) + } + return { + ...failedStructuredAgentSessionStart(), + startKey: child.generation, + failure: failure.fact, + retry: failure.retry === 'safe' + } +} + +/** The deadline keeps its own outcome; a broken adapter contract fails the start; anything else + * stopping it is cancellation. */ +function stoppedStructuredAgentSessionStart( + reason: unknown +): Extract { + if (isStructuredAgentSessionStartupExpired(reason)) { + return { ...failedStructuredAgentSessionStart(), expired: true } + } + if (isStructuredAgentSessionStartupContractBreach(reason)) { + return { + ...failedStructuredAgentSessionStart(), + failure: agentSessionFailureFact('providerStartFailed') + } + } + return { ...failedStructuredAgentSessionStart(), aborted: true } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts index 6f718e6f44fe..eec28f5b17b6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts @@ -8,6 +8,7 @@ import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session/a import type { AgentSessionRecordStore } from '../../runtime/agent-session/records/agent-session-record-store' import { openTestAgentSessionRecordStore } from '../../runtime/agent-session/records/agent-session-record-store-test-harness' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { structuredAgentSessionHeldStarts } from './structured-agent-session-held-start.test-support' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { HOST_TEST_NOW as NOW, @@ -46,6 +47,7 @@ let generation = 0 let exitOnStart = false let dispatch: Mock let frames: AgentSessionSubscribeEvent[] = [] +const starts = structuredAgentSessionHeldStarts(() => host) function exitBeforeProof(): Promise { return host.handleAdapterEvent({ @@ -105,6 +107,7 @@ beforeEach(async () => { resetHostTestOperationIds() generation = 0 frames = [] + starts.clear() exitOnStart = false dispatch = vi.fn(async () => ({ state: 'admitted' as const })) store = await openTestAgentSessionRecordStore(root) @@ -112,9 +115,10 @@ beforeEach(async () => { agents: NO_STRUCTURED_AGENTS, logger: createStructuredAgentSessionLogger(), store, - adapter: { + // The adapter answers its never-proved opens with its own child's exit. + adapter: starts.wrap({ open: () => new Promise(() => {}), - acquire: vi.fn(async ({ fence, spawnToken }) => { + acquire: starts.held(async ({ fence, spawnToken }) => { const child = { process: { hostId: 'local', @@ -150,7 +154,7 @@ beforeEach(async () => { cancelTurn: vi.fn(async () => ({ cancelled: true })), answerPrompt: vi.fn(async () => undefined), setOption: vi.fn(async () => undefined) - }, + }), journalDatabase: openTestJournalHostDatabase(root), claimKeyId: 'key-1', mintSpawnToken: () => `spawn-${generation + 1}`, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts new file mode 100644 index 000000000000..8a5323179a18 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure.ts @@ -0,0 +1,253 @@ +// The adapter that holds the protocol connection decides whether a failed start may run again; +// the host reads only this verdict, never provider error classes. + +import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { findInErrorChain } from '../../provider-process/provider-error-chain' +import { + providerExecutableMissing, + providerLaunchFolderMissing +} from '../../provider-process/provider-executable-missing' +import { providerRequestTimeoutOf } from '../../provider-process/provider-request-timeout' +import { argumentProblemOf } from '../structured-agent-arguments-error' +import { agentCliInstallationProblemOf } from '../agent-cli-installation-error' +import { + AgentSessionAcquisitionRefusal, + isAgentSessionPreSpawnError +} from './structured-agent-session-adapter-errors' +import { + AgentSessionAcquisitionExitProvenError, + AgentSessionAcquisitionExitUnprovenError, + AgentSessionAcquisitionRootExitObservedError +} from './structured-agent-session-acquisition-errors' +import { + launchPreparationDiagnosticOf, + providerExitObserved, + providerStartupFailureFact, + startupAcquisitionRefusal, + withObservedProviderExit +} from './structured-agent-session-failure-text' + +/** `safe`: the agent gave no answer before its output ended, or declared its answer temporary. */ +export type AgentSessionStartRetry = 'safe' | 'never' + +/** An adapter's reading of one error in the chain: the agent's answer, final or declared + * temporary; Orca's own record of its transport ending with no answer once output drained; or + * an SDK's report of that ending, which tells less than Orca's record of it. */ +export type AgentSessionStartAnswer = ( + cause: Error +) => 'final' | 'temporary' | 'exited' | 'reported' | undefined + +export class AgentSessionStartFailure extends Error { + readonly fact: SubmissionRejectionFact + + constructor( + cause: unknown, + readonly retry: AgentSessionStartRetry, + fact?: SubmissionRejectionFact + ) { + super(cause instanceof Error ? cause.message : String(cause), { cause }) + this.name = 'AgentSessionStartFailure' + this.fact = fact ?? providerStartupFailureFact(cause) + } +} + +/** One verdict from every observation an adapter kept of a failed start: the operation's errors, + * a failed result seen for it and Orca's own record of its process ending, earliest first. It + * may run again only if every observation allows it; the most specific one explains it, ties + * going to the earliest. `exited`: the adapter saw its process end on its own, which words even + * a final answer as the agent's own failure to start without changing whether it may run again. */ +export function agentSessionStartFailure( + evidence: unknown, + answer?: AgentSessionStartAnswer, + exited = false +): AgentSessionStartFailure { + const observed = Array.isArray(evidence) + ? evidence.filter((cause, index) => cause != null && evidence.indexOf(cause) === index) + : [evidence] + const verdict = observed.find((cause) => cause instanceof AgentSessionStartFailure) + if (verdict) { + return verdict + } + const readings = observed.map((cause) => startDisposition(cause, answer)) + const ranks = observed.map((cause, index) => explanationRank(cause, readings[index].answers)) + const selected = ranks.indexOf(Math.min(...ranks)) + const error = observed[selected] + // An adapter's own exit error words the start as the agent's failure, as an observed exit does. + const worded = + (exited || readings[selected].exited) && error instanceof Error + ? withObservedProviderExit(new Error(error.message, { cause: error })) + : error + const retry = readings.every((reading) => reading.retry === 'safe') ? 'safe' : 'never' + return new AgentSessionStartFailure(error, retry, providerStartupFailureFact(worded)) +} + +/** Most specific first: what the user can act on, the request's deadline, the agent's answer, + * Orca's record of its exit, an SDK's report of that exit, then anything else. */ +function explanationRank(cause: unknown, answers: ReadonlySet): number { + const rank = [ + agentCliInstallationProblemOf(cause), + startupAcquisitionRefusal(cause), + providerExecutableMissing(cause), + providerLaunchFolderMissing(cause), + launchPreparationDiagnosticOf(cause) ?? argumentProblemOf(cause), + providerRequestTimeoutOf(cause), + ['final', 'temporary', 'never', 'safe'].some((read) => answers.has(read)) || + providerResourcePressure(cause), + answers.has('exited') || providerExitObserved(cause), + answers.has('reported') + ].findIndex(Boolean) + return rank === -1 ? 9 : rank +} + +/** The adapter's verdict on an error where it read an answer or its own exit; otherwise none. */ +export function agentSessionStartVerdict( + error: unknown, + answer: AgentSessionStartAnswer +): AgentSessionStartFailure | undefined { + if (error instanceof AgentSessionStartFailure) { + return error + } + const answered = findInErrorChain(error, (cause) => + answer(cause) === undefined ? undefined : true + ) + return answered ? agentSessionStartFailure(error, answer) : undefined +} + +/** Acquisition failures keep at their top the exit proofs and refusals lease bookkeeping reads; + * the adapter's verdict goes beneath them, so a final answer outlives its cleanup's proof. */ +export function agentSessionAcquireFailure( + error: unknown, + answer: AgentSessionStartAnswer +): unknown { + if ( + error instanceof AgentSessionAcquisitionRootExitObservedError || + error instanceof AgentSessionAcquisitionExitProvenError + ) { + const verdict = agentSessionStartVerdict(error.cause, answer) + if (!verdict || verdict === error.cause) { + return error + } + return error instanceof AgentSessionAcquisitionRootExitObservedError + ? new AgentSessionAcquisitionRootExitObservedError(verdict, error.deathEvidenceKind) + : new AgentSessionAcquisitionExitProvenError(verdict) + } + if ( + error instanceof AgentSessionAcquisitionExitUnprovenError || + error instanceof AgentSessionAcquisitionRefusal || + isAgentSessionPreSpawnError(error) + ) { + return error + } + return agentSessionStartVerdict(error, answer) ?? error +} + +/** How each starting acquisition ended before the host committed it ready, with the adapter's + * verdict. An acquisition is admitted when it starts and retired once nothing can read its ending: + * its start's owner finished, it failed, or its session was released, disposed or closed. A + * retired acquisition records nothing, so an exit after ready never leaves an entry behind. */ +export class AgentSessionStartEndings { + private readonly endings = new Map< + string, + { generation: string; failure?: AgentSessionStartFailure } + >() + + /** The session's new acquisition; an earlier one's ending can no longer be read. */ + admit(sessionId: string, generation: string): void { + this.endings.set(sessionId, { generation }) + } + + /** The first ending of an admitted acquisition is its verdict; later reports change nothing. */ + record(sessionId: string, generation: string, failure: AgentSessionStartFailure): void { + const admitted = this.endings.get(sessionId) + if (admitted?.generation === generation) { + admitted.failure ??= failure + } + } + + get(sessionId: string, generation: string): AgentSessionStartFailure | undefined { + const ended = this.endings.get(sessionId) + return ended?.generation === generation ? ended.failure : undefined + } + + /** Without a generation, retires whatever the session holds. */ + retire(sessionId: string, generation?: string): void { + if (generation === undefined || this.endings.get(sessionId)?.generation === generation) { + this.endings.delete(sessionId) + } + } + + clear(): void { + this.endings.clear() + } +} + +/** The verdict for an error no adapter converted, such as an acquisition the host wrapped. */ +export function agentSessionStartRetry(error: unknown): AgentSessionStartRetry { + return error instanceof AgentSessionStartFailure ? error.retry : startDisposition(error).retry +} + +/** A failed acquisition replays only once its exit is proven and the adapter's verdict allows it. */ +export function retryableAgentSessionAcquisition(error: unknown): boolean { + return ( + (error instanceof AgentSessionAcquisitionExitProvenError || + error instanceof AgentSessionAcquisitionRootExitObservedError) && + agentSessionStartRetry(error) === 'safe' + ) +} + +/** A final answer anywhere in the chain wins. Cleanup's exit proof alone says nothing about the + * agent's answer, so only the agent's own exit, a declared temporary answer or a launch short of + * host resources permits one more run. */ +function startDisposition( + error: unknown, + answer?: AgentSessionStartAnswer +): { + retry: AgentSessionStartRetry + exited: boolean + answers: Set | AgentSessionStartRetry> +} { + const answers = new Set | AgentSessionStartRetry>() + findInErrorChain(error, (cause) => { + answers.add(cause instanceof AgentSessionStartFailure ? cause.retry : answer?.(cause)) + // An adapter's verdict already covers the errors beneath it. + return cause instanceof AgentSessionStartFailure ? true : undefined + }) + if ( + providerRequestTimeoutOf(error) || + argumentProblemOf(error) || + agentCliInstallationProblemOf(error) || + providerExecutableMissing(error) || + providerLaunchFolderMissing(error) || + findInErrorChain(error, (cause) => + cause instanceof AggregateError || + cause instanceof AgentSessionAcquisitionExitUnprovenError || + cause instanceof AgentSessionAcquisitionRefusal || + isAgentSessionPreSpawnError(cause) + ? true + : undefined + ) || + answers.has('never') || + answers.has('final') + ) { + return { retry: 'never', exited: false, answers } + } + const exited = answers.has('exited') || answers.has('reported') + const safe = + exited || + answers.has('safe') || + answers.has('temporary') || + providerExitObserved(error) || + providerResourcePressure(error) + return { retry: safe ? 'safe' : 'never', exited, answers } +} + +/** The host ran short of processes or files; the same launch can run once that passes. */ +export function providerResourcePressure(error: unknown): boolean { + return ( + findInErrorChain(error, (cause) => + 'code' in cause && ['EAGAIN', 'EMFILE', 'ENFILE'].some((code) => code === cause.code) + ? true + : undefined + ) === true + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts new file mode 100644 index 000000000000..df84c31604a2 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-lifecycle-perf.test.ts @@ -0,0 +1,590 @@ +import { getEventListeners } from 'node:events' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { createAcpAgentConnection } from '../../acp/acp-agent-connection' +import { openAcpAdapterRig } from '../../acp/acp-structured-adapter.test-support' +import { readAntigravitySignInLine } from '../../antigravity/antigravity-sign-in-url' +import { findInErrorChain } from '../../provider-process/provider-error-chain' +import { createFakePipedChild } from '../../../shared/__fixtures__/fake-spawned-child' +import { closeProviderTimelineRigs } from '../agent-session-timeline/provider-timeline-assembler-test-support' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session/lifecycle/agent-session-mutation-envelope' +import { + adapterAtPublishFor, + fakeClaude, + identityFor, + tick +} from '../../claude/structured/claude-structured-session-test-support' +import { + sessionId as piSessionId, + setup as setupPi +} from '../../pi/rpc-session-adapter.test-fixture' +import { withObservedProviderExit } from './structured-agent-session-failure-text' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' +import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { + adapterFor, + fakeCodex, + THREAD_ID +} from '../../codex/codex-structured-session-adapter-fixture' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session/records/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { + claudeAndCodexAgents, + claudeAndCodexRouter +} from './structured-agent-session-adapter-router-test-support' +import type * as ExitObservation from './structured-agent-session-exit-observation' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' +import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' +import { restTestSend } from './structured-agent-session-rest-test-rig' +import type * as StartFailure from './structured-agent-session-start-failure' + +// Every adapter's startup receipts and every acquisition's root-exit receipt, to count what they +// still hold after each start, whichever adapter or sink created them. +const created = vi.hoisted(() => ({ + endings: new Array(), + rootExits: new Array() +})) +const retainedReceipts = () => + created.endings.reduce((sum, endings) => sum + endings['endings'].size, 0) +vi.mock('./structured-agent-session-start-failure', async (original) => { + const actual = await original() + class CountedEndings extends actual.AgentSessionStartEndings { + constructor() { + super() + created.endings.push(this) + } + } + return { ...actual, AgentSessionStartEndings: CountedEndings } +}) +vi.mock('./structured-agent-session-exit-observation', async (original) => { + const actual = await original() + class CountedRootExit extends actual.StructuredAgentSessionRootExitReceipt { + constructor() { + super() + created.rootExits.push(this) + } + } + return { ...actual, StructuredAgentSessionRootExitReceipt: CountedRootExit } +}) + +const CALLER = { callerKey: 'client-1' } +const CEILING_MS = 20_000 +type Cycle = 'ready' | 'final' | 'retried' | 'expired' | 'stopped' +const CYCLES: readonly Cycle[] = ['ready', 'final', 'retried', 'expired', 'stopped'] +const SCRIPTS: Record = { + ready: ['ok'], + final: ['refuse'], + retried: ['hang', 'ok'], + expired: ['hang'], + stopped: ['hang'] +} + +let root: string +beforeEach(async () => { + vi.useFakeTimers({ + toFake: ['Date', 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval'] + }) + root = await mkdtemp(join(tmpdir(), 'orca-start-lifecycle-')) + resetHostTestOperationIds() + created.endings.length = 0 + created.rootExits.length = 0 +}) +afterEach(async () => { + try { + await closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) + } finally { + vi.useRealTimers() + vi.restoreAllMocks() + } +}) + +/** Runs `count` rounds of each start outcome through the production host, router and Codex adapter. */ +async function runStarts(count: number) { + // Each thread/start or thread/resume takes the next scripted answer. + const answers: ('ok' | 'refuse' | 'hang')[] = [] + const threadAnswer = () => { + const answer = answers.shift() + if (answer === 'refuse') { + throw new CodexAppServerRequestError('thread/start', -32600, 'invalid', 'invalid') + } + return answer === 'hang' + ? new Promise(() => {}) + : { thread: { id: THREAD_ID }, model: 'gpt-live', reasoningEffort: 'medium' } + } + const codex = fakeCodex({ 'thread/start': threadAnswer, 'thread/resume': threadAnswer }) + let host: StructuredAgentSessionHost | undefined + const store = await openTestAgentSessionRecordStore(join(root, `starts-${count}`)) + const adapter = adapterFor(codex, {}, [], { + // Once a thread exists, each new child resumes it. + resolveLaunch: async () => ({ + command: 'codex', + args: ['app-server'], + cwd: '/work/repo', + codexHome: null, + resumeThreadId: store.getRecord(SESSION)?.providerHandleChain.length ? THREAD_ID : null + }), + onEvent: (event) => { + if (event.type === 'ended' && 'cause' in event) { + void host?.handleAdapterEvent(event) + } + } + }) + const other = adapterFor(fakeCodex()) + const router = claudeAndCodexRouter({ claude: other, codex: adapter }, async () => { + await adapter.closeAll() + await other.closeAll() + }) + const dispatch = vi.fn(async () => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD_ID, + turnId: `turn-${dispatch.mock.calls.length}`, + ordinal: dispatch.mock.calls.length + } + })) + // Turn work is not under test; every start-path call reaches the production router. + Object.assign(router, { dispatch }) + host = new StructuredAgentSessionHost({ + agents: claudeAndCodexAgents({ claude: other, codex: adapter }), + logger: createStructuredAgentSessionLogger(), + store, + adapter: router, + journalDatabase: openTestJournalHostDatabase(join(root, `starts-${count}`)), + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${codex.connections.length}`, + startupLimits: { ceilingMs: CEILING_MS }, + idleSweep: { intervalMs: 60 * 60_000, idleMs: 60 * 60_000 }, + now: () => NOW + }) + const { runtimeState } = host.collaboratorsForTests() + const spies = { + opens: vi.spyOn(router, 'open'), + // The router reaches every registered adapter by exact generation. + codexEndingReads: vi.spyOn(adapter, 'startupEnding'), + otherEndingReads: vi.spyOn(other, 'startupEnding'), + codexRetirements: vi.spyOn(adapter, 'retireStartup'), + otherRetirements: vi.spyOn(other, 'retireStartup'), + timersScheduled: vi.spyOn(globalThis, 'setTimeout') + } + // Abort handles the host still holds, and the listeners on their signals. + const abortSignals = () => + [...runtimeState.acquireAborts['inFlight'].values()].flatMap((waits) => + [...waits].map(({ controller }) => controller.signal) + ) + const abortListeners = () => + abortSignals().reduce((sum, signal) => sum + getEventListeners(signal, 'abort').length, 0) + const held = () => ({ + timers: vi.getTimerCount(), + abortListeners: abortListeners(), + abortHandles: abortSignals().length, + trackedStarts: runtimeState.startupAttempts['open'].size, + owners: runtimeState.startupAttempts['owners'].size, + flights: runtimeState.startFlights['flying'].size, + bufferedExits: runtimeState.startFlights['exits'].size, + receipts: retainedReceipts(), + rootExitListeners: created.rootExits.reduce((sum, exit) => sum + exit['listeners'].size, 0), + liveChildren: codex.connections.filter((connection) => !connection.closed).length, + // Host work still tracked or queued after its caller returned. + trackedTasks: host!['tasks']['attaching'].size, + queuedTasks: host!['tasks']['chains'].size + }) + const eventually = (assertion: () => unknown) => + vi.waitFor(assertion, { timeout: 10_000, interval: 0 }) + + const submission = async (id: string) => + (await host!.journalSnapshot(SESSION)).submissions.find((row) => row.clientMessageId === id) + const failureRows = async () => + (await host!.journalSnapshot(SESSION)).items.filter( + (item) => item.body.kind === 'status' && item.body.tone === 'error' + ).length + const phase = () => host!.collaboratorsForTests().sessions.get(SESSION)?.child?.phase + const exitChild = (connection = codex.connections.at(-1)!) => { + connection.closed = true + const exit = new CodexAppServerExitError('codex app-server connection ended') + // A duplicate terminal report must change nothing. + connection.handlers.onExit?.(exit, { expected: false }) + connection.handlers.onExit?.(exit, { expected: false }) + } + + // Work the starts did, counted from a ready chat. + const work = async (): Promise> => ({ + ...Object.fromEntries( + Object.entries(spies).map(([field, spy]) => [field, spy.mock.calls.length]) + ), + spawns: codex.connections.length, + failureRows: await failureRows() + }) + + answers.push('ok') + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + await eventually(() => expect(phase()).toBe('ready')) + const begun = await work() + const after: Record[]> = { + ready: [], + final: [], + retried: [], + expired: [], + stopped: [] + } + let rows = 0 + // What a Stopped start holds while its child's cleanup is still running. + const stopping: ReturnType[] = [] + // What one start holds while its child is still answering. + const starting: ReturnType[] = [] + for (let round = 0; round < count; round++) { + for (const cycle of CYCLES) { + // The chat's ready child dies, so the next message starts a new one as the script says. + if (phase() === 'ready') { + exitChild() + await eventually(() => expect(phase()).toBeUndefined()) + } + await host.flushStreamedEvents(SESSION) + rows = await failureRows() + const spawned = codex.connections.length + answers.push(...SCRIPTS[cycle]) + const send = restTestSend(cycle, store.getRecord(SESSION)!.lease.runtimeFence) + expect(await host.send(CALLER, send), `${cycle} ${round}`).toMatchObject({ ok: true }) + if (cycle === 'retried') { + await eventually(() => expect(answers).toEqual(['ok'])) + starting.push(held()) + exitChild() + } + if (cycle === 'expired') { + await eventually(() => expect(answers).toEqual([])) + await vi.advanceTimersByTimeAsync(CEILING_MS) + } + if (cycle === 'stopped') { + await eventually(() => expect(answers).toEqual([])) + // The user stops the start while its open waits; the child's cleanup is held, then ends. + const connection = codex.connections.at(-1)! + const close = connection.close + const cleanup = Promise.withResolvers() + connection.close = async () => { + await cleanup.promise + return close() + } + const fence = store.getRecord(SESSION)!.lease.runtimeFence + const stop = { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: SESSION, + fields: {} + }) + } + const stopped = host.cancel(CALLER, { envelope: stop }) + await vi.advanceTimersByTimeAsync(100) + stopping.push(held()) + cleanup.resolve() + expect(await stopped).toMatchObject({ ok: true }) + } + // The start's owner decides it: the message runs on a ready child, or one row says why not. + const started = cycle === 'ready' || cycle === 'retried' + await eventually(async () => { + await vi.advanceTimersByTimeAsync(100) + expect(runtimeState.startFlights.has(SESSION)).toBe(false) + expect(phase(), `${cycle} ${round}`).toBe(started ? 'ready' : undefined) + expect((await submission(send.envelope.clientOperationId))?.dispatchState).toBe( + started ? 'accepted' : 'rejected' + ) + expect(await failureRows()).toBe(rows + (cycle === 'final' || cycle === 'expired' ? 1 : 0)) + }) + expect(codex.connections).toHaveLength(spawned + (cycle === 'retried' ? 2 : 1)) + await host.flushStreamedEvents(SESSION) + await vi.advanceTimersByTimeAsync(2_000) + after[cycle].push(held()) + } + } + const done = await work() + await host.close(SESSION, 'evict') + await host.flushAllStreamedEvents() + await router.closeAll() + vi.restoreAllMocks() + const counts = Object.fromEntries( + Object.entries(done).map(([field, count]) => [field, count - begun[field]]) + ) + return { after, starting, stopping, counts } +} + +it('returns every start resource to baseline and does constant work per start, at N and 10N', async () => { + const n = await runStarts(1) + const tenN = await runStarts(10) + for (const cycle of CYCLES) { + const [first] = n.after[cycle] + // Nothing a finished start owned survives it: no owner, abort handle, tracked attempt, flight, + // buffered exit, adapter receipt, root-exit listener or tracked task, and no exited child left + // running. + expect(first, cycle).toMatchObject({ + abortHandles: 0, + trackedStarts: 0, + owners: 0, + flights: 0, + bufferedExits: 0, + receipts: 0, + rootExitListeners: 0, + liveChildren: cycle === 'ready' || cycle === 'retried' ? 1 : 0, + trackedTasks: 0, + queuedTasks: 0 + }) + // Timers and abort listeners held after each start do not grow with the starts before it. + for (const held of tenN.after[cycle]) { + expect(held, cycle).toEqual(first) + } + } + // While a start waits on its child it holds a fixed set, however many starts came before it: + // one deadline timer, the flight's and the owner's abort handles, one tracked attempt, owner, + // receipt and root-exit subscription. + expect(n.starting[0]).toMatchObject({ + timers: n.after.ready[0].timers + 1, + abortHandles: 2, + trackedStarts: 1, + owners: 1, + flights: 1, + receipts: 1, + rootExitListeners: 1 + }) + for (const held of tenN.starting) { + expect(held).toEqual(n.starting[0]) + } + // A Stop releases the start's owner at once; while its child's cleanup is held, that cleanup is + // the one task still tracked and queued, however many Stops came before it. + expect(n.stopping[0]).toMatchObject({ + owners: 0, + flights: 0, + abortHandles: 0, + receipts: 0, + trackedTasks: 1, + queuedTasks: 1, + liveChildren: 1 + }) + for (const held of tenN.stopping) { + expect(held).toEqual(n.stopping[0]) + } + // Per round of five starts: six spawns and opens (the retried start opens twice) and a failure + // row for the final and the expired start; a Stopped start reads no ending. Router reads and + // retirements reach the registered adapters a fixed number of times per open, whatever the + // starts before it. + expect(n.counts).toMatchObject({ + spawns: 6, + opens: 6, + codexEndingReads: 4, + otherEndingReads: 4, + codexRetirements: 6, + otherRetirements: 6, + failureRows: 2 + }) + for (const [field, count] of Object.entries(n.counts)) { + expect(tenN.counts[field], field).toBe(count * 10) + } +}) + +/** Starts an ACP agent that prints a sign-in line Orca cannot read, `count` times. */ +async function refuseAcpStarts(count: number) { + const connections: ReturnType[] = [] + const refusals = new Map, Error>() + const rig = await openAcpAdapterRig({ + deps: { + connect: (launch, options) => { + const child = Object.assign(createFakePipedChild(), { + pid: 9_999_999, + stdout: new PassThrough(), + stdin: new PassThrough(), + stderr: new PassThrough(), + kill: () => true + }) + // Orca's cleanup ends stdin; the agent then exits. + child.stdin.once('finish', () => child.emit('exit', 0, null)) + const connection: ReturnType = createAcpAgentConnection( + launch, + { + ...options, + onClose: (error) => { + if (!refusals.has(connection)) { + refusals.set(connection, error) + } + options.onClose?.(error) + }, + signIn: { read: readAntigravitySignInLine } + }, + () => child + ) + connections.push(connection) + child.stdout.write('Open the following link to authenticate the ACP server: not-a-url\n') + return connection + } + } + }) + const retained: number[] = [] + try { + for (let index = 0; index < count; index++) { + const error = await rig.acquire({ waitForStart: false }).catch((caught: unknown) => caught) + const connection = connections[index] + // The verdict is final and carries this connection's own first refusal, not a later close. + const verdict = findInErrorChain(error, (cause) => + cause instanceof AgentSessionStartFailure ? cause : undefined + ) + expect(verdict?.retry).toBe('never') + expect(verdict?.cause).toBe(refusals.get(connection)) + expect(connection.lostBeforeExit).toBe(refusals.get(connection)) + expect(connection.exited).toBe(true) + retained.push(retainedReceipts()) + } + return { spawns: connections.length, retained } + } finally { + await rig.adapter.closeAll() + await closeProviderTimelineRigs() + } +} + +it('ACP: each refused start keeps its own cause and leaves no receipt or child, at N and 10N', async () => { + vi.useRealTimers() + for (const count of [10, 100]) { + const { spawns, retained } = await refuseAcpStarts(count) + expect(spawns).toBe(count) + expect(retained).toEqual(Array(count).fill(0)) + } +}) + +/** What a real Claude or Pi adapter keeps from `count` starts of each ending, retired by the + * start's owner as the host does: a ready start, and one whose agent exited before its open. */ +async function adapterStarts( + agent: 'claude' | 'pi', + count: number +): Promise<{ left: Record[]; spawns: number }> { + const harness = agent === 'claude' ? await claudeStarts() : await piStarts() + const left: Record[] = [] + try { + for (let round = 0; round < count; round++) { + for (const exitsFirst of [false, true]) { + const generation = await harness.acquire() + if (exitsFirst) { + harness.exitChild() + } + const opened = await harness.open(generation).then( + () => 'ready', + (error: unknown) => error + ) + if (exitsFirst) { + // The open answers with the ending the exit left, a verdict the owner can act on. + expect(opened).toBeInstanceOf(AgentSessionStartFailure) + } else { + expect(opened).toBe('ready') + } + harness.adapter.retireStartup?.(harness.sessionId, generation) + // Retired by its owner, even while a ready start's session stays live. + const receipts = retainedReceipts() + await harness.adapter.closeSession(harness.sessionId) + await harness.settle() + left.push({ receipts, ...harness.held() }) + } + } + return { left, spawns: harness.spawns() } + } finally { + await harness.adapter.closeAll() + await closeProviderTimelineRigs() + } +} + +async function claudeStarts() { + const claude = fakeClaude() + const adapter = adapterAtPublishFor(claude) + const sessionId = identityFor().sessionId + return { + adapter, + sessionId, + acquire: async () => + (await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'perf' })) + .acquisitionGeneration, + open: (acquisitionGeneration: string) => + adapter.open({ + sessionId, + fence: 7, + acquisitionGeneration, + signal: new AbortController().signal + }), + exitChild: () => { + const connection = claude.connections.at(-1)! + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(withObservedProviderExit(new Error('claude exited')), { + expected: false + }) + }, + settle: async () => { + adapter.acknowledgeSessionRelease?.(sessionId) + await tick() + }, + held: () => ({ + sessions: adapter['sessions'].size, + acquisitions: adapter['acquisitions']['attempts'].size, + exits: adapter['exits'].size + }), + spawns: () => claude.connections.length + } +} + +async function piStarts() { + const pi = await setupPi() + const { adapter, input, connections } = pi + await adapter.closeSession(piSessionId) + return { + adapter, + sessionId: piSessionId, + acquire: async () => (await adapter.acquire(input)).acquisitionGeneration, + open: (acquisitionGeneration: string) => + adapter.open({ + sessionId: piSessionId, + fence: 7, + acquisitionGeneration, + signal: new AbortController().signal + }), + exitChild: () => connections.at(-1)!.exit(), + settle: async () => { + await adapter.drainObservedExits() + adapter.acknowledgeSessionRelease(piSessionId) + }, + held: () => ({ + sessions: adapter['sessions'].size, + rootExitListeners: connections.reduce( + (sum, connection) => sum + connection['exitListeners'].length, + 0 + ) + }), + spawns: () => connections.length + } +} + +it.each(['claude', 'pi'] as const)( + '%s: each start retires its receipt and exit subscription, at N and 10N', + async (agent) => { + vi.useRealTimers() + const n = await adapterStarts(agent, 10) + const tenN = await adapterStarts(agent, 100) + const settled = + agent === 'claude' + ? { sessions: 0, acquisitions: 0, exits: 0 } + : { sessions: 0, rootExitListeners: 0 } + for (const held of [...n.left, ...tenN.left]) { + expect(held).toEqual({ receipts: 0, ...settled }) + } + expect(tenN.spawns - (agent === 'pi' ? 1 : 0)).toBe(10 * (n.spawns - (agent === 'pi' ? 1 : 0))) + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-provider-answer.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-provider-answer.ts new file mode 100644 index 000000000000..d62e6f736bd2 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-provider-answer.ts @@ -0,0 +1,142 @@ +import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { recordAgentSessionProviderHandle } from '../../runtime/agent-session/ownership/agent-session-provider-handle-transition' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import type { StructuredAgentSessionProviderOpenResult } from './structured-agent-session-provider-open-contract' +import type { StructuredAgentSessionProviderStartedContext } from './structured-agent-session-provider-started' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' +import { + AgentSessionStartFailure, + agentSessionStartFailure +} from './structured-agent-session-start-failure' +import { + applyStructuredRewindRecovery, + readStructuredRewindRecovery, + type StructuredRewindRecoveryEvidence +} from './structured-rewind-recovery' + +// What a starting child's open answered, read and recorded before the start is ready. + +export type OpenedProvider = StructuredAgentSessionProviderOpenResult & { + sessionId: string + fence: number + acquisitionGeneration: string +} + +/** Orca could not record or apply what the open answered; nothing about the agent says retry. */ +export function refusedStart(): AgentSessionStartFailure { + return new AgentSessionStartFailure( + new Error('Orca could not record what the agent answered at its start'), + 'never', + agentSessionFailureFact('hostStopped') + ) +} + +/** Reconcile uncertain rewind before delivery; persistent uncertainty fails startup with the + * adapter's verdict on its read, or as final when Orca's own reading failed. */ +export async function readRewindAtStart( + context: StructuredAgentSessionProviderStartedContext, + event: OpenedProvider, + signal?: AbortSignal +): Promise { + const { sessionId } = event + const journal = context.sessions.get(sessionId)?.journal + if (!journal) { + return null + } + try { + return await readStructuredRewindRecovery( + context.deps, + sessionId, + journal, + structuredAgentSessionConversationFence(context.deps.store, sessionId), + context.deps.adapter, + signal, + recordWithStartedHandle(context, event) + ) + } catch (error) { + rewindRecoveryFailed(context, sessionId, error, signal) + return agentSessionStartFailure(error) + } +} + +/** Predict the committed handle for rewind cuts; if recording is refused, retain the existing record. */ +function recordWithStartedHandle( + context: StructuredAgentSessionProviderStartedContext, + { sessionId, fence, link }: OpenedProvider +): AgentSessionRecord | null { + const record = context.deps.store.getRecord(sessionId) + try { + return record && link + ? recordAgentSessionProviderHandle({ record, fence, link, now: context.now() }) + : record + } catch { + return record + } +} + +export async function applyRewindAtStart( + context: StructuredAgentSessionProviderStartedContext, + sessionId: string, + session: StructuredAgentSessionHostSession, + evidence: StructuredRewindRecoveryEvidence +): Promise { + try { + await applyStructuredRewindRecovery( + context.deps, + sessionId, + session.journal, + structuredAgentSessionConversationFence(context.deps.store, sessionId), + evidence + ) + return true + } catch (error) { + rewindRecoveryFailed(context, sessionId, error) + return false + } +} + +function rewindRecoveryFailed( + context: StructuredAgentSessionProviderStartedContext, + sessionId: string, + error: unknown, + signal?: AbortSignal +): void { + if (!signal?.aborted) { + context.deps.logger.warn('settling a rewind in doubt at the start failed', { + scope: 'rewind-recovery', + sessionId, + error + }) + } +} + +/** Persist the opened handle before delivery so future starts can resume; false if no handle is proven. */ +export async function recordStartedHandle( + context: StructuredAgentSessionProviderStartedContext, + event: OpenedProvider +): Promise { + const { store, logger } = context.deps + const { link } = event + if (!link) { + // Only an already-proven handle permits open without a new link. + return store.getRecord(event.sessionId)?.lease.provenHandleLinkId != null + } + try { + const committed = await store.transitionHandoff(event.sessionId, (record) => + recordAgentSessionProviderHandle({ record, fence: event.fence, link, now: context.now() }) + ) + // Inherit only a committed new context; a resumed CLI keeps its own later title. + if (event.newConversation ?? (link.origin === 'created' || link.origin === 'forked')) { + context.deps.nameProviderConversation?.(committed) + } + return true + } catch (error) { + logger.warn('recording the handle a started provider answered with failed', { + scope: 'provider-started-handle', + sessionId: event.sessionId, + error + }) + return false + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts index 853a0d61e175..f7b2c4a416a3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry-recorded.test.ts @@ -1,6 +1,16 @@ +import canonicalFilenameRefusal from '../../codex/app-server/fixtures/start-resume-canonical-filename-refusal.json' +import sdkExit from '../../claude/fixtures/startup-sdk-process-exit.json' +import { CodexAppServerUnsupportedError } from '../../codex/app-server/codex-app-server-session' import { PassThrough } from 'node:stream' import { describe, expect, it, vi } from 'vitest' -import { AcpAgentError, AcpInvalidResponseError, AcpRpcError } from '../../acp/acp-errors' +import { + AcpAgentError, + AcpAgentExitedError, + AcpConnectionClosedError, + AcpInvalidResponseError, + AcpRequestTimeoutError, + AcpRpcError +} from '../../acp/acp-errors' import { acpRpcEnvelopeSchema, settleAcpResponse } from '../../acp/acp-rpc-envelope' import { ClaudeControlRequestError, @@ -10,7 +20,14 @@ import { createCodexAppServerRecordDispatcher } from '../../codex/app-server/cod import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' import { JsonlRpcPeer, JsonlRpcResponseError } from '../../jsonl-rpc/peer' import { withProviderRequestTimeout } from '../../provider-process/provider-request-timeout' -import { withProviderSpawnAbsence } from '../../provider-process/provider-executable-missing' +import { + withMissingProviderExecutable, + withProviderSpawnAbsence +} from '../../provider-process/provider-executable-missing' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session/failures/agent-session-failure-words' +import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' +import { JsonlRpcAgentExitError } from '../../jsonl-rpc/agent-connection' import { AgentSessionAcquisitionExitProvenError, AgentSessionAcquisitionExitUnprovenError, @@ -19,11 +36,32 @@ import { import { AgentSessionAcquisitionRefusal } from './structured-agent-session-adapter-errors' import { withObservedProviderExit } from './structured-agent-session-failure-text' import { - retryableStructuredAgentSessionAcquisition, - retryableStructuredAgentSessionStart -} from './structured-agent-session-start-retry' + agentSessionAcquireFailure, + agentSessionStartFailure, + retryableAgentSessionAcquisition, + type AgentSessionStartAnswer, + type AgentSessionStartFailure +} from './structured-agent-session-start-failure' +import { codexStartFailure, codexStartupAnswer } from '../../codex/app-server/codex-startup-outcome' +import { acpStartFailure, acpStartupAnswer } from '../../acp/acp-startup-outcome' +import { piStartFailure, piStartupAnswer } from '../../pi/rpc-startup-outcome' +import { claudeStartFailure, claudeStartupAnswer } from '../../claude/claude-startup-failure' import captures from './__fixtures__/startup-protocol-rejections.json' +// Each adapter reads only its own answers; the shared rule then gives the generic verdict. +const adapterAnswer: AgentSessionStartAnswer = (cause) => + [codexStartupAnswer, acpStartupAnswer, piStartupAnswer, claudeStartupAnswer] + .map((answer) => answer(cause)) + .find((verdict) => verdict !== undefined) + +function retryableStructuredAgentSessionStart(error: unknown): boolean { + return agentSessionStartFailure(error, adapterAnswer).retry === 'safe' +} + +function retryableStructuredAgentSessionAcquisition(error: unknown): boolean { + return retryableAgentSessionAcquisition(agentSessionAcquireFailure(error, adapterAnswer)) +} + function expectExitEvidenceRequired(error: unknown) { expect(retryableStructuredAgentSessionStart(error)).toBe(true) expect(retryableStructuredAgentSessionAcquisition(error)).toBe(false) @@ -40,16 +78,19 @@ function expectExitEvidenceRequired(error: unknown) { ).toBe(false) } -function expectFinalWithExitEvidence(error: Error) { +function expectFinalWithExitEvidence(error: unknown) { expect(retryableStructuredAgentSessionStart(error)).toBe(false) expect( retryableStructuredAgentSessionAcquisition(new AgentSessionAcquisitionExitProvenError(error)) ).toBe(false) + if (!(error instanceof Error)) { + throw new Error('expected Error') + } expect(retryableStructuredAgentSessionStart(withObservedProviderExit(error))).toBe(false) } describe('recorded command rejection wrappers at the startup classifier', () => { - it('keeps recorded Claude permission rejection retryable without a typed startup refusal', async () => { + it('keeps a recorded Claude control rejection final, including after exit', async () => { const capture = captures.claude // SDK rejection is supplied here; this does not exercise the SDK's wire parser. const error = await runClaudeControl(capture.request.request.subtype, () => @@ -60,10 +101,10 @@ describe('recorded command rejection wrappers at the startup classifier', () => subtype: capture.request.request.subtype, message: capture.response.response.error }) - expectExitEvidenceRequired(error) + expectFinalWithExitEvidence(error) }) - it('keeps the recorded Codex invalid thread/read answer retryable outside resume', async () => { + it('keeps a recorded Codex answer final outside resume too', async () => { const capture = captures.codex const pending = Promise.withResolvers() const failed = pending.promise.catch((caught: unknown) => caught) @@ -85,10 +126,10 @@ describe('recorded command rejection wrappers at the startup classifier', () => code: capture.response.error.code, providerDiagnostic: { text: capture.response.error.message, audience: 'person' } }) - expectExitEvidenceRequired(error) + expectFinalWithExitEvidence(error) }) - it('keeps the recorded Pi model rejection retryable without a typed startup refusal', async () => { + it('keeps a recorded Pi rejection final, including after exit', async () => { const capture = captures.pi const input = new PassThrough() const output = new PassThrough() @@ -108,7 +149,7 @@ describe('recorded command rejection wrappers at the startup classifier', () => command: capture.response.command, message: capture.response.error }) - expectExitEvidenceRequired(error) + expectFinalWithExitEvidence(error) } finally { peer.close() input.destroy() @@ -116,7 +157,7 @@ describe('recorded command rejection wrappers at the startup classifier', () => } }) - it('keeps the recorded ACP -32603 retryable regardless of its InvalidRequestError data', async () => { + it('keeps a recorded ACP permanent rejection final', async () => { const capture = captures.acp const pending = Promise.withResolvers() const failed = pending.promise.catch((caught: unknown) => caught) @@ -126,7 +167,7 @@ describe('recorded command rejection wrappers at the startup classifier', () => expect(error).toBeInstanceOf(AcpAgentError) expect(error).toMatchObject(capture.response.error) expect(diagnose).not.toHaveBeenCalled() - expectExitEvidenceRequired(error) + expectFinalWithExitEvidence(error) }) it.each([ @@ -167,8 +208,8 @@ describe('recorded command rejection wrappers at the startup classifier', () => expectFinalWithExitEvidence(error) }) - it.each([-32600, -32603])('keeps unknown Codex resume answers retryable: %s', (code) => { - expectExitEvidenceRequired( + it.each([-32600, -32603])('keeps unknown Codex resume answers final: %s', (code) => { + expectFinalWithExitEvidence( new CodexAppServerRequestError('thread/resume', code, 'Server is temporarily unavailable') ) }) @@ -193,3 +234,304 @@ describe('recorded command rejection wrappers at the startup classifier', () => ) }) }) + +// The wire capture has no error.data, so the error is built without one. +it('keeps the recorded canonical rollout filename refusal final on the first answer', () => { + const { method, code, message } = canonicalFilenameRefusal + expectFinalWithExitEvidence(new CodexAppServerRequestError(method, code, message, message)) +}) + +it('preserves an actual SDK nonzero exit through initialize control cleanup', async () => { + const recorded = sdkExit.records[0].initializationError + if (!recorded) { + throw new Error('missing recorded initialization error') + } + const cause = Object.assign(new Error(recorded.message), recorded) + const error = await runClaudeControl('initialize', () => Promise.reject(cause)).catch( + (error) => error + ) + expect(error.cause).toBe(cause) + expectExitEvidenceRequired(error) +}) + +it('keeps unsupported Codex methods final after process exit too', () => { + expectFinalWithExitEvidence(new CodexAppServerUnsupportedError('method unsupported')) +}) + +describe('positive provider startup retry contracts', () => { + const codex = (method: string, code: number, message: string, data?: unknown) => + new CodexAppServerRequestError(method, code, message, message, data) + it.each([ + codex('initialize', -32001, 'Server overloaded; retry later.'), + codex('thread/start', -32600, 'Server is draining; retry after reconnecting', { + reason: 'serverShuttingDown' + }), + codex('thread/resume', -32600, 'Server is draining; retry after reconnecting', { + reason: 'serverShuttingDown' + }), + codex('thread/start', -32600, 'Workspace change in progress; retry when it completes'), + codex( + 'thread/resume', + -32600, + 'thread abc is closing; retry thread/resume after the thread is closed' + ), + ...['thread/start', 'thread/resume'].flatMap((method) => [ + codex(method, -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode: 'Timeout' + }), + ...[undefined, 503, 403].map((statusCode) => + codex(method, -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode: 'RequestFailed', + statusCode, + detail: 'Failed to load cloud config bundle (workspace-managed policies).' + }) + ) + ]), + codex( + 'thread/start', + -32603, + 'error creating thread: storage failed: thread-store retryable error: busy' + ), + new AcpAgentError(-32603, 'server unavailable', { errorName: 'ServerUnavailable' }), + new AcpAgentError(-32603, 'catalog not ready', { errorName: 'CatalogNotReady' }) + ])('retries only after separately proving absence: $message', expectExitEvidenceRequired) + + it.each([ + ...['thread/start', 'thread/resume'].flatMap((method) => [ + ...['Auth', 'InvalidBundle', 'Internal', 'RequestFailed'].map((errorCode) => + codex(method, -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode + }) + ), + codex(method, -32600, 'failed to load configuration', { + reason: 'cloudConfigBundle', + errorCode: 'Timeout', + action: 'relogin' + }) + ]), + codex('thread/start', -32603, 'error creating thread: outcome unknown'), + codex('thread/start', -32603, 'error creating thread: internal error'), + codex('thread/resume', -32600, 'please retry this unknown answer') + ])( + 'does not infer retry from an error code or a generic message: $message', + expectFinalWithExitEvidence + ) + + it.each([ + ['control_request_failed', false], + ['process_exited_nonzero', true], + ['process_killed_by_signal', true], + ['error_result', false], + ['initialize_timeout', false], + ['executable_not_found', false], + ['executable_launch_failed', false], + ['aborted', false], + ['unreadable', false], + ['future_unknown_tag', false], + [undefined, false] + ])('preserves SDK %s through the control wrapper', async (errorClass, retry) => { + const cause = Object.assign(new Error('SDK error'), errorClass ? { errorClass } : {}) + const error = await runClaudeControl('initialize', () => Promise.reject(cause)).catch( + (error) => error + ) + expect(error.cause).toBe(cause) + expect(retryableStructuredAgentSessionStart(error)).toBe(retry) + if (!retry) { + expectFinalWithExitEvidence(error) + } + }) + + it('preserves Codex error.data at the response boundary', async () => { + const pending = Promise.withResolvers() + const result = pending.promise.catch((error) => error) + const dispatcher = createCodexAppServerRecordDispatcher({ + handlers: {}, + writeResponse: vi.fn(), + onProtocolFailure: vi.fn() + }) + dispatcher.addPending(1, { method: 'thread/start', ...pending, timer: undefined }) + const data = { reason: 'serverShuttingDown' } + dispatcher.dispatch({ + id: 1, + error: { code: -32600, message: 'Server is draining; retry after reconnecting', data } + }) + const error = await result + if (!(error instanceof CodexAppServerRequestError)) { + throw new Error('expected a Codex request error') + } + expect(error.data).toBe(data) + expect(Object.keys(error)).not.toContain('data') + expectExitEvidenceRequired(error) + }) +}) + +describe('one verdict from every observation of a failed start', () => { + const recordedSdkExit = async (exitCode: number) => { + const recorded = sdkExit.records[0].initializationError + if (!recorded) { + throw new Error('missing recorded initialization error') + } + // The Debian lane's supervised spawn ENOENT: the recorded SDK shape, with the supervisor's 127. + const cause = Object.assign(new Error(`Claude Code process exited with code ${exitCode}`), { + ...recorded, + exitCode + }) + return runClaudeControl('initialize', () => Promise.reject(cause)).catch( + (error: unknown) => error + ) + } + const claudeOwnExit = (message: string) => withObservedProviderExit(new Error(message)) + const recordedCodexRefusal = async () => { + const pending = Promise.withResolvers() + const failed = pending.promise.catch((caught: unknown) => caught) + const dispatcher = createCodexAppServerRecordDispatcher({ + handlers: {}, + writeResponse: vi.fn(), + onProtocolFailure: vi.fn() + }) + dispatcher.addPending(captures.codex.request.id, { + method: captures.codex.request.method, + ...pending, + timer: undefined + }) + dispatcher.dispatch(captures.codex.response) + return failed + } + const loadTimeout = new AcpRequestTimeoutError('session/load', 300_000) + type Row = { + name: string + verdict: (evidence: unknown[]) => AgentSessionStartFailure + observations: () => Promise<[explains: unknown, other: unknown]> + retry: 'safe' | 'never' + kind: string + } + const rows: Row[] = [ + { + name: "Claude: Orca's exit that found no CLI over the SDK's exit 127 report", + verdict: (evidence) => claudeStartFailure(evidence), + observations: async () => [ + withMissingProviderExecutable( + claudeOwnExit('claude stream-json exited (code 127): spawn claude ENOENT') + ), + await recordedSdkExit(127) + ], + retry: 'never', + kind: 'cliMissing' + }, + { + name: "Claude: Orca's plain exit record over the SDK's report of it", + verdict: (evidence) => claudeStartFailure(evidence), + observations: async () => [ + claudeOwnExit('claude stream-json exited (code 1)'), + await recordedSdkExit(1) + ], + retry: 'safe', + kind: 'providerStartFailed' + }, + { + name: 'Claude: a recorded control rejection over its exit', + verdict: (evidence) => claudeStartFailure(evidence), + observations: async () => [ + await runClaudeControl(captures.claude.request.request.subtype, () => + Promise.reject(new Error(captures.claude.response.response.error)) + ).catch((caught: unknown) => caught), + claudeOwnExit('claude stream-json exited (code 1)') + ], + retry: 'never', + kind: 'startFailed' + }, + { + name: 'Codex: a recorded refusal over its drained exit, worded as that exit', + verdict: (evidence) => codexStartFailure(evidence, true), + observations: async () => [ + await recordedCodexRefusal(), + new CodexAppServerExitError('codex app-server exited') + ], + retry: 'never', + kind: 'providerStartFailed' + }, + { + name: 'Codex: a declared temporary answer beside its drained exit', + verdict: (evidence) => codexStartFailure(evidence, true), + observations: async () => [ + new CodexAppServerRequestError('initialize', -32001, 'Server overloaded; retry later.'), + new CodexAppServerExitError('codex app-server exited') + ], + retry: 'safe', + kind: 'providerStartFailed' + }, + { + name: 'ACP: a load deadline over the exit seen while its output drained', + verdict: (evidence) => acpStartFailure(evidence, true), + observations: async () => [loadTimeout, new AcpAgentExitedError('agent exited')], + retry: 'never', + kind: 'startFailed' + }, + { + name: "ACP: a deleted launch folder over the transport's closed summary", + verdict: (evidence) => acpStartFailure(evidence), + observations: async () => [ + withProviderSpawnAbsence(new AcpAgentExitedError('agent exited'), 'launchFolder'), + new AcpConnectionClosedError() + ], + retry: 'never', + kind: 'launchFolderMissing' + }, + { + name: 'Pi: a recorded rejection over its exit', + verdict: (evidence) => piStartFailure(evidence), + observations: async () => [ + new JsonlRpcResponseError(captures.pi.response.command, captures.pi.response.error), + new JsonlRpcAgentExitError('Pi exited while starting') + ], + retry: 'never', + kind: 'startFailed' + }, + { + name: 'Pi: its exit explains an unexplained failure, which still keeps the start final', + verdict: (evidence) => piStartFailure(evidence), + observations: async () => [ + new JsonlRpcAgentExitError('Pi exited while starting'), + new Error('Invalid agent RPC response') + ], + retry: 'never', + kind: 'providerStartFailed' + } + ] + + it.each(rows)('$name', async ({ verdict, observations, retry, kind }) => { + const [explains, other] = await observations() + // Precedence is the policy's, not the order the adapter saw them in. + for (const evidence of [ + [explains, other], + [other, explains] + ]) { + const failure = verdict(evidence) + expect(failure.cause).toBe(explains) + expect(failure.retry).toBe(retry) + expect(failure.fact.kind).toBe(kind) + } + }) + + it("keeps a deadline's method, duration and words over the exit around it", () => { + const failure = acpStartFailure([new AcpAgentExitedError('agent exited'), loadTimeout], true) + const timedOut = agentSessionFailureFact('startFailed', { + requestTimeout: { method: 'session/load', timeoutMs: 300_000 } + }) + expect(failure.fact).toEqual(timedOut) + expect(agentSessionFailureWords(failure.fact, { surface: 'rejection' })).toEqual( + agentSessionFailureWords(timedOut, { surface: 'rejection' }) + ) + }) + + it('still lets an unanswered exit run again when every observation is that exit', async () => { + const own = claudeOwnExit('claude stream-json exited (code 1)') + expect(claudeStartFailure([own, await recordedSdkExit(1)]).retry).toBe('safe') + expect(piStartFailure([new JsonlRpcAgentExitError('Pi exited while starting')]).retry).toBe( + 'safe' + ) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts deleted file mode 100644 index 93dec21f6507..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-retry.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' -import { ClaudeControlRequestError } from '../../claude/claude-agent-sdk-control-requests' -import { AcpAgentError, AcpRpcError } from '../../acp/acp-errors' -import { JsonlRpcResponseError } from '../../jsonl-rpc/peer' -import { findInErrorChain } from '../../provider-process/provider-error-chain' -import { - providerExecutableMissing, - providerLaunchFolderMissing -} from '../../provider-process/provider-executable-missing' -import { providerRequestTimeoutOf } from '../../provider-process/provider-request-timeout' -import { argumentProblemOf } from '../structured-agent-arguments-error' -import { agentCliInstallationProblemOf } from '../agent-cli-installation-error' -import { - AgentSessionAcquisitionExitProvenError, - AgentSessionAcquisitionRootExitObservedError, - AgentSessionAcquisitionExitUnprovenError, - AgentSessionAcquisitionRefusal, - isAgentSessionPreSpawnError -} from './structured-agent-session-adapter' -import { providerExitObserved } from './structured-agent-session-failure-text' - -function finalAgentStartAnswer(error: Error): boolean { - if (error instanceof AcpRpcError && !(error instanceof AcpAgentError)) { - return true - } - if (!(error instanceof AcpAgentError || error instanceof CodexAppServerRequestError)) { - return false - } - if (error.code === -32601 || error.code === -32602) { - return true - } - if ( - !(error instanceof CodexAppServerRequestError) || - error.method !== 'thread/resume' || - error.code !== -32600 - ) { - return false - } - const message = - error.providerDiagnostic?.text ?? - error.message.replace(/^codex app-server thread\/resume failed: /, '') - return /^(?:invalid (?:thread|session) id:|thread not found(?::|$)|no rollout found for thread id )/.test( - message - ) -} - -/** Only positive provider evidence retries; an Orca fault or a typed refusal stays final. */ -export function retryableStructuredAgentSessionStart(error: unknown): boolean { - if ( - providerRequestTimeoutOf(error) || - argumentProblemOf(error) || - agentCliInstallationProblemOf(error) || - providerExecutableMissing(error) || - providerLaunchFolderMissing(error) - ) { - return false - } - const final = findInErrorChain(error, (cause) => - cause instanceof AggregateError || - cause instanceof AgentSessionAcquisitionExitUnprovenError || - cause instanceof AgentSessionAcquisitionRefusal || - isAgentSessionPreSpawnError(cause) || - finalAgentStartAnswer(cause) - ? true - : undefined - ) - if (final) { - return false - } - return ( - providerExitObserved(error) || - findInErrorChain(error, (cause) => - cause instanceof CodexAppServerRequestError || - cause instanceof ClaudeControlRequestError || - cause instanceof AcpAgentError || - cause instanceof JsonlRpcResponseError || - ('code' in cause && ['EAGAIN', 'EMFILE', 'ENFILE'].some((code) => code === cause.code)) - ? true - : undefined - ) === true - ) -} - -export function retryableStructuredAgentSessionAcquisition(error: unknown): boolean { - return ( - (error instanceof AgentSessionAcquisitionExitProvenError || - error instanceof AgentSessionAcquisitionRootExitObservedError) && - retryableStructuredAgentSessionStart(error) - ) -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts index eafb3d3084b0..7901fe7326a9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-settlement-host.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as TimerPromises from 'node:timers/promises' import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' import { withProviderRequestTimeout } from '../../provider-process/provider-request-timeout' import { AgentSessionAcquisitionRefusal } from './structured-agent-session-adapter' import { agentSessionFailureWords } from '../../../shared/agent-session/failures/agent-session-failure-words' @@ -8,6 +9,8 @@ import { agentSessionFailureFact } from '../../../shared/agent-session/failures/ import { DISPATCH_REJECTED_CANCELLED } from '../../../shared/agent-session/requests/structured-agent-session-dispatch-rejection' import * as startFlight from './structured-agent-session-start-flight' import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from './structured-agent-session-send-settlement' +import { agentSessionStartFailure } from './structured-agent-session-start-failure' +import { withObservedProviderExit } from './structured-agent-session-failure-text' import { CALLER, attach, @@ -38,6 +41,7 @@ vi.mock('node:timers/promises', async (importOriginal) => { let host: ReturnType['host'] let acquire: ReturnType['acquire'] let dispatch: ReturnType['dispatch'] +const openFailures = new Map void>() const openGates = new Map void>() const eventually = (assertion: () => void | Promise) => vi.waitFor(assertion, { timeout: 10_000, interval: 0 }) @@ -52,8 +56,10 @@ beforeEach(async () => { await host.close(SESSION, 'evict') acquire.mockClear() openGates.clear() + openFailures.clear() host.deps.adapter.open = ({ acquisitionGeneration }) => - new Promise((resolve) => { + new Promise((resolve, reject) => { + openFailures.set(acquisitionGeneration!, reject) openGates.set(acquisitionGeneration!, () => resolve({ reportedOptions: {} })) }) }) @@ -116,6 +122,13 @@ async function endAttempt() { reason: 'exited while starting', cause: 'unexpected-exit' }) + // As the adapter: its open answers with this acquisition's own unanswered exit. + openFailures.get(child.generation)?.( + agentSessionStartFailure( + withObservedProviderExit(new Error('provider exited before open answered')) + ) + ) + openFailures.delete(child.generation) await eventually(() => expect(session.child).toBeNull()) } @@ -165,10 +178,15 @@ describe('final logical startup settlement', () => { await lifetime.failStartingChild( SESSION, sessions.get(SESSION)!.child!, - withProviderRequestTimeout(new CodexAppServerRequestError('initialize', -1, 'deadline'), { - method: 'initialize', - timeoutMs: 60_000 - }), + codexStartFailure( + withProviderRequestTimeout( + new CodexAppServerRequestError('initialize', -1, 'deadline'), + { + method: 'initialize', + timeoutMs: 60_000 + } + ) + ), 60_000 ) await expect(flight.outcome).resolves.toMatchObject({ ok: false, aborted: true }) @@ -258,10 +276,12 @@ describe('final logical startup settlement', () => { await lifetime.failStartingChild( SESSION, sessions.get(SESSION)!.child!, - withProviderRequestTimeout(new CodexAppServerRequestError('initialize', -1, 'deadline'), { - method: 'initialize', - timeoutMs: 60_000 - }), + codexStartFailure( + withProviderRequestTimeout(new CodexAppServerRequestError('initialize', -1, 'deadline'), { + method: 'initialize', + timeoutMs: 60_000 + }) + ), 60_000 ) await eventually(() => expect(conversationDelivery.loop.isRunning(SESSION)).toBe(false)) @@ -369,6 +389,7 @@ describe('final logical startup settlement', () => { expect(runtimeState.currentEventSink(SESSION)).toBeUndefined() acquire.mockClear() openGates.clear() + openFailures.clear() const other = 'unrelated-chat' const unrelated = hostTestAttachParams(null, { providerHandle: undefined, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-contract.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-contract.ts index 5ae27374e38b..962060db4402 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-contract.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-contract.ts @@ -49,6 +49,22 @@ export function isStructuredAgentSessionStartupExpired(reason: unknown): boolean return reason instanceof Error && reason.name === 'StructuredAgentSessionStartupExpiredError' } +/** How long after an acquisition's own root exit its adapter may take to answer for the start. + * Every adapter's post-exit work (output drain, initialize-answer join) finishes well inside it. */ +export const STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS = 5_000 + +/** The adapter gave no outcome within the grace after its process exited: a broken contract. */ +export class StructuredAgentSessionStartupContractError extends Error { + constructor(agent: string) { + super(`the ${agent} adapter gave no startup outcome after its process exited`) + this.name = 'StructuredAgentSessionStartupContractError' + } +} + +export function isStructuredAgentSessionStartupContractBreach(reason: unknown): boolean { + return reason instanceof Error && reason.name === 'StructuredAgentSessionStartupContractError' +} + export type StructuredAgentSessionAcquisitionReceipt = Pick< StructuredAgentSessionStartupAttempt, 'attemptId' | 'fence' | 'spawnToken' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts index eddaae0b0256..597a1eb4972b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host-test-support.ts @@ -33,6 +33,8 @@ import { import { StructuredAgentSessionHost } from './structured-agent-session-host' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import type { StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' +import type { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import { testAdapterStartupEnding } from './structured-agent-session-held-start.test-support' export const CALLER = { callerKey: 'client-1' } @@ -46,8 +48,12 @@ export let acquire: Mock export let dispatch: Mock export let closeSession: Mock> let setOption: Mock +export const providerOpen = vi.fn>() export const providerStarted: Mock = vi.fn() export const recordLiveListing: Mock = vi.fn() +const openFailures = new Map void>() +const endings = new Map() + const openGates = new Map void>() export async function proveOpen(model = 'default'): Promise { @@ -57,6 +63,23 @@ export async function proveOpen(model = 'default'): Promise { openGates.delete(generation) } +/** As an adapter whose open rejects with its own verdict for the current acquisition. */ +export function failOpen(error: Error): void { + const generation = `generation-${acquire.mock.calls.length}` + openFailures.get(generation)?.(error) + openFailures.delete(generation) +} + +/** As an adapter seeing the current acquisition's own root exit. */ +export function observeRootExit(): void { + acquire.mock.calls.at(-1)?.[0].events?.observeRootExit?.() +} + +/** As an adapter recording the current acquisition's unanswered death before it was ready. */ +export function recordStartupEnding(): void { + endings.set(`generation-${acquire.mock.calls.length}`, testAdapterStartupEnding()) +} + export function acquisition(input: StructuredAgentSessionAcquireInput): AgentSessionAcquisition { return { process: { @@ -90,16 +113,12 @@ async function startHost({ startupLimits, supportsGoals }: StartupHostOptions = logger: createStructuredAgentSessionLogger(), store, adapter: { - open: ({ acquisitionGeneration }) => - new Promise((resolve) => { - openGates.set(acquisitionGeneration!, (model) => - resolve({ reportedOptions: { model, reportedPermissionMode: 'ask' } }) - ) - }), + open: providerOpen, acquire, dispatch, closeSession, releaseAcquisition: vi.fn(async () => true), + startupEnding: (_sessionId, generation) => endings.get(generation), cancelTurn: vi.fn(async () => ({ cancelled: false })), answerPrompt: vi.fn(async () => undefined), setOption @@ -126,6 +145,25 @@ async function startHost({ startupLimits, supportsGoals }: StartupHostOptions = ...(startupLimits === undefined ? {} : { startupLimits }), now: () => NOW }) + const handleEvent = host.handleAdapterEvent.bind(host) + host.handleAdapterEvent = async (event) => { + // As an adapter: its own ending of this exact acquisition answers the open and the start. + const ending = + event.type === 'ended' && event.cause === 'unexpected-exit' && event.acquisitionGeneration + ? testAdapterStartupEnding(event.failure) + : undefined + if (ending && event.type === 'ended') { + endings.set(event.acquisitionGeneration, ending) + } + const result = await handleEvent(event) + if (event.type === 'ended' && event.acquisitionGeneration) { + openFailures.get(event.acquisitionGeneration)?.( + ending ?? new Error('provider exited before open answered') + ) + openFailures.delete(event.acquisitionGeneration) + } + return result + } } export async function restartWith( @@ -134,6 +172,7 @@ export async function restartWith( ): Promise { await host.close(SESSION, 'evict') await host.flushAllStreamedEvents() + providerOpen.mockClear() acquire.mockImplementation(next) closeSession.mockClear() await startHost(options) @@ -149,7 +188,18 @@ export function registerStartupAttemptHostTests(): void { resetHostTestOperationIds() providerStarted.mockReset() recordLiveListing.mockReset() + openFailures.clear() openGates.clear() + endings.clear() + providerOpen.mockReset().mockImplementation( + ({ acquisitionGeneration }) => + new Promise((resolve, reject) => { + openFailures.set(acquisitionGeneration!, reject) + openGates.set(acquisitionGeneration!, (model) => + resolve({ reportedOptions: { model, reportedPermissionMode: 'ask' } }) + ) + }) + ) acquire = vi.fn(async (input) => acquisition(input)) dispatch = vi.fn(async () => ({ state: 'accepted' as const, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts index bd53451c45bc..602ec51cc9de 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { CodexAppServerRequestError } from '../../codex/app-server/codex-app-server-request-error' +import { codexStartFailure } from '../../codex/app-server/codex-startup-outcome' import { withProviderRequestTimeout } from '../../provider-process/provider-request-timeout' import { AgentSessionAcquisitionRefusal, @@ -17,7 +18,10 @@ import { HOST_TEST_THREAD as THREAD, hostTestOperationId } from './structured-agent-session-host-test-data' -import type { StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' +import { + STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS, + type StructuredAgentSessionStartupLimits +} from './structured-agent-session-startup-attempt-contract' import { CALLER, accept, @@ -30,12 +34,16 @@ import { host, proveOpen, providerStarted, + providerOpen, publishFirst, restartWith as restartHostWith, stop, store, submission, - registerStartupAttemptHostTests + registerStartupAttemptHostTests, + failOpen, + observeRootExit, + recordStartupEnding } from './structured-agent-session-startup-attempt-host-test-support' let startupLimits: Partial | undefined @@ -321,6 +329,103 @@ describe('one logical start across physical attempts', () => { return event } + it.each([ + [false, false], + [false, true], + [true, false], + [true, true] + ] as const)( + 'retains a final open answer (exit first: %s, typed refusal: %s)', + async (exitFirst, typed) => { + await restartWith(publishFirst) + const open = Promise.withResolvers() + providerOpen.mockReturnValueOnce(open.promise) + const id = await heldBehindStart('one request') + await eventually(() => expect(providerOpen).toHaveBeenCalledTimes(1)) + const refusal = typed + ? new AgentSessionAcquisitionRefusal('saved conversation missing') + : new CodexAppServerRequestError('thread/resume', -32600, 'canonical filename required') + if (exitFirst) { + await endAttempt() + } + open.reject(refusal) + if (!exitFirst) { + await vi.advanceTimersByTimeAsync(20) + } + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + await vi.advanceTimersByTimeAsync(2_000) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + } + ) + + it('fails a start whose adapter never answers after its own exit, before the ceiling and without a replacement', async () => { + await restartWith(publishFirst) + let signal: AbortSignal | undefined + providerOpen.mockImplementationOnce((input) => { + signal = input.signal + return new Promise(() => {}) + }) + const id = await heldBehindStart('one request') + // The adapter saw its process exit, but neither its open nor its exit report ever settles. + observeRootExit() + await vi.advanceTimersByTimeAsync(STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect(await submission(id)).toMatchObject( + agentSessionFailureWords(agentSessionFailureFact('providerStartFailed'), { + agentName: 'Codex', + surface: 'rejection' + }) + ) + expect(signal?.aborted).toBe(true) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('recovers when the child dies after its open answered but before the start was ready', async () => { + await restartWith(publishFirst) + const id = await heldBehindStart('one request') + // The adapter's own record of this acquisition ending lands before the host commits ready. + recordStartupEnding() + await proveOpen() + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + await proveOpen() + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(dispatch).toHaveBeenCalledTimes(1) + }) + + it('keeps an initialize deadline that rejects after the exit it raced', async () => { + await restartWith(publishFirst) + const open = Promise.withResolvers() + providerOpen.mockReturnValueOnce(open.promise) + const id = await heldBehindStart('deadline after exit') + await endAttempt() + const requestTimeout = { method: 'initialize', timeoutMs: 60_000 } + open.reject(withProviderRequestTimeout(new Error('initialize timed out'), requestTimeout)) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect((await submission(id))?.rejection).toMatchObject({ requestTimeout }) + expect(acquire).toHaveBeenCalledTimes(1) + }) + + it('reconnects after a declared-temporary open answer and delivers the held message once', async () => { + await restartWith(publishFirst) + providerOpen.mockRejectedValueOnce( + codexStartFailure( + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) + ) + ) + const id = await heldBehindStart('recover') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + await proveOpen() + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(dispatch).toHaveBeenCalledTimes(1) + }) + function goal() { const change = { kind: 'set' as const, objective: 'finish startup' } return host.changeThreadGoal(CALLER, { @@ -413,15 +518,7 @@ describe('one logical start across physical attempts', () => { closeSession.mockResolvedValue(false) const second = await heldBehindStart('unproven process') const child = host.collaboratorsForTests().sessions.get(SESSION)!.child! - const { lifetime } = host.collaboratorsForTests() - await expect( - lifetime.failStartingChild( - SESSION, - child, - new CodexAppServerRequestError('thread/start', -1, 'refused'), - 1 - ) - ).rejects.toThrow('eviction failed') + failOpen(codexStartFailure(new CodexAppServerRequestError('thread/start', -1, 'refused'))) await eventually(async () => expect((await submission(second))?.dispatchState).toBe('rejected')) expect(host.collaboratorsForTests().sessions.get(SESSION)?.child).toBe(child) expect(acquire).toHaveBeenCalledTimes(1) @@ -490,22 +587,29 @@ describe('one logical start across physical attempts', () => { ) expect(acquire).toHaveBeenCalledTimes(2) expect(await submission(second)).toMatchObject(HOST_STOPPED) + acquire.mockClear() + providerOpen.mockImplementationOnce(() => new Promise(() => {})) + const third = await heldBehindStart('ceiling inside the exit join') + await vi.advanceTimersByTimeAsync(900) + await endAttempt() + await vi.advanceTimersByTimeAsync(150) + await eventually(async () => expect((await submission(third))?.dispatchState).toBe('rejected')) + expect(await submission(third)).toMatchObject(HOST_STOPPED) + expect(acquire).toHaveBeenCalledTimes(1) }) it('keeps an elapsed provider-step deadline final and preserves its words', async () => { await restartWith(publishFirst) const first = await heldBehindStart('initialize deadline') - const { sessions, lifetime } = host.collaboratorsForTests() const requestTimeout = { method: 'initialize', timeoutMs: 60_000 } await vi.advanceTimersByTimeAsync(60_000) - await lifetime.failStartingChild( - SESSION, - sessions.get(SESSION)!.child!, - withProviderRequestTimeout( - new CodexAppServerRequestError('initialize', -1, 'deadline'), - requestTimeout - ), - 60_000 + failOpen( + codexStartFailure( + withProviderRequestTimeout( + new CodexAppServerRequestError('initialize', -1, 'deadline'), + requestTimeout + ) + ) ) await eventually(async () => expect((await submission(first))?.dispatchState).toBe('rejected')) expect(await submission(first)).toMatchObject( @@ -575,12 +679,7 @@ describe('one logical start across physical attempts', () => { const expectedMessages = [later] expect(await submission(later)).toMatchObject({ dispatchState: 'pending' }) if (failsWrite) { - await eventually(() => - expect(host.collaboratorsForTests().conversationDelivery.loop.isRunning(SESSION)).toBe( - false - ) - ) - expect(await submission(later)).toMatchObject({ dispatchState: 'pending' }) + // The failed start's row not being written holds up no later message. restoreWrite?.() expectedMessages.push(await accept('after the journal recovered')) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-mint.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-mint.ts new file mode 100644 index 000000000000..fa10dd7d0a3c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-mint.ts @@ -0,0 +1,36 @@ +import { randomUUID } from 'node:crypto' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session/journal/agent-session-journal-identity' +import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' +import type { AgentChatPermissionChoice } from '../../../shared/agent-chat-permission-mode' +import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import type { StructuredAgentSessionStartupAttempt } from './structured-agent-session-startup-attempt-contract' + +export function mintStructuredAgentSessionStartupAttempt(input: { + record: AgentSessionRecord + identity: AgentSessionJournalIdentity + spawnToken: string + permissionMode?: AgentChatPermissionChoice + events?: StructuredAgentSessionEventSink + signal?: AbortSignal + startupOptions?: Readonly> + attemptId?: string +}): StructuredAgentSessionStartupAttempt { + const { record } = input + const selected = input.startupOptions ?? record.options + const options = record.provider === 'claude' ? { fastMode: 'false', ...selected } : selected + return { + attemptId: input.attemptId ?? randomUUID(), + identity: input.identity, + fence: record.lease.runtimeFence, + spawnToken: input.spawnToken, + launch: { + location: record.location, + accountHome: record.accountHome, + ...(record.launchDirectory === undefined ? {} : { launchDirectory: record.launchDirectory }) + }, + ...(options ? { options } : {}), + ...(input.permissionMode ? { permissionMode: input.permissionMode } : {}), + ...(input.events ? { events: input.events } : {}), + ...(input.signal ? { signal: input.signal } : {}) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts index 2a862589f8ee..ab65901e2ec7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt.ts @@ -1,86 +1,43 @@ -import { randomUUID } from 'node:crypto' -import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' import { waitForStructuredAgentSessionStart, type StructuredAgentSessionStartReadiness } from './structured-agent-session-start-readiness' export type { StructuredAgentSessionStartReadiness } from './structured-agent-session-start-readiness' -import type { AgentSessionJournalIdentity } from '../../../shared/agent-session/journal/agent-session-journal-identity' -import type { AgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record' -import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' -import type { AgentChatPermissionChoice } from '../../../shared/agent-chat-permission-mode' import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' import { sameProviderChild } from './structured-agent-session-provider-child' import { + STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS, STRUCTURED_AGENT_SESSION_STARTUP_LIMITS, type StructuredAgentSessionStartupAttempt, type StructuredAgentSessionStartupLimits } from './structured-agent-session-startup-attempt-contract' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import type { SubmissionRejectionFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { + measureStartup, + startupContractFailure, + startupDue, + trackedStartup, + type StructuredAgentSessionExpiredStartup, + type StructuredAgentSessionStartupProgress, + type StructuredAgentSessionStartupSettled, + type StructuredAgentSessionTrackedStartup +} from './structured-agent-session-startup-tracking' -export function mintStructuredAgentSessionStartupAttempt(input: { - record: AgentSessionRecord - identity: AgentSessionJournalIdentity - spawnToken: string - permissionMode?: AgentChatPermissionChoice - events?: StructuredAgentSessionEventSink - signal?: AbortSignal - startupOptions?: Readonly> - attemptId?: string -}): StructuredAgentSessionStartupAttempt { - const { record } = input - const selected = input.startupOptions ?? record.options - const options = record.provider === 'claude' ? { fastMode: 'false', ...selected } : selected - return { - attemptId: input.attemptId ?? randomUUID(), - identity: input.identity, - fence: record.lease.runtimeFence, - spawnToken: input.spawnToken, - launch: { - location: record.location, - accountHome: record.accountHome, - ...(record.launchDirectory === undefined ? {} : { launchDirectory: record.launchDirectory }) - }, - ...(options ? { options } : {}), - ...(input.permissionMode ? { permissionMode: input.permissionMode } : {}), - ...(input.events ? { events: input.events } : {}), - ...(input.signal ? { signal: input.signal } : {}) - } -} - -export type StructuredAgentSessionExpiredStartup = { - sessionId: string - attemptId: string - child: StructuredAgentSessionProviderChildIdentity | null -} - -export type StructuredAgentSessionStartupSettled = { - agent: AgentSessionJournalIdentity['agent'] - outcome: 'ready' | 'ceiling' | 'ended' - durationMs: number -} +export { mintStructuredAgentSessionStartupAttempt } from './structured-agent-session-startup-attempt-mint' -export type StructuredAgentSessionStartupProgress = { - spawned: () => void -} +export type { + StructuredAgentSessionExpiredStartup, + StructuredAgentSessionStartupProgress, + StructuredAgentSessionStartupSettled +} from './structured-agent-session-startup-tracking' -type Tracked = { - attemptId: string - attempt: StructuredAgentSessionStartupAttempt | null - child: StructuredAgentSessionProviderChildIdentity | null - /** Absolute ceiling starts before any startup wait. */ - registeredAt: number - spawnedAt: number | null - timer: ReturnType | null - expired: boolean - /** Defer expiry reporting until aborted spawn settles; successful readiness overrides it. */ - expiry: StructuredAgentSessionStartupSettled | null - failure?: SubmissionRejectionFact - retry?: boolean - readiness: PromiseWithResolvers<'ready' | 'ended'> -} +type Tracked = StructuredAgentSessionTrackedStartup export class StructuredAgentSessionStartupAttempts { private readonly open = new Map() + /** The open settling each starting child's outcome; while it runs, the child's end waits for it. */ + private readonly owners = new Map() private disposed = false private readonly limits: StructuredAgentSessionStartupLimits private readonly now: () => number @@ -89,6 +46,8 @@ export class StructuredAgentSessionStartupAttempts { private readonly deps: { /** Expire outside serialization so blocked provider startup cannot delay cancellation. */ expire: (expired: StructuredAgentSessionExpiredStartup) => void + /** The adapter answered nothing within the grace after its exit; cancel like expiry. */ + breach?: (breached: StructuredAgentSessionExpiredStartup & { agent: string }) => void settled?: (settled: StructuredAgentSessionStartupSettled) => void limits?: Partial /** Use elapsed time; host clocks may be pinned. */ @@ -109,31 +68,9 @@ export class StructuredAgentSessionStartupAttempts { return this.now() - registeredAt >= this.limits.ceilingMs } - childFailed( - sessionId: string, - child: StructuredAgentSessionProviderChildIdentity, - failure: SubmissionRejectionFact, - retry: boolean - ): void { - const tracked = this.open.get(sessionId) - if (tracked?.child && sameProviderChild(tracked.child, child)) { - Object.assign(tracked, { failure, retry }) - } - } - private register(sessionId: string, attemptId: string, registeredAt = this.now()): Tracked { this.end(sessionId) - const tracked: Tracked = { - attemptId, - attempt: null, - child: null, - registeredAt, - spawnedAt: null, - timer: null, - expired: false, - expiry: null, - readiness: Promise.withResolvers() - } + const tracked = trackedStartup(attemptId, registeredAt) this.open.set(sessionId, tracked) if (!this.disposed) { this.schedule(sessionId, tracked) @@ -151,6 +88,9 @@ export class StructuredAgentSessionStartupAttempts { ? begun : this.register(sessionId, attempt.attemptId) tracked.attempt = attempt + tracked.stopListening ??= attempt.events?.onRootExit?.(() => + this.rootExited(sessionId, tracked) + ) const current = (): boolean => this.open.get(sessionId) === tracked && !this.disposed return { spawned: () => { @@ -175,6 +115,39 @@ export class StructuredAgentSessionStartupAttempts { } } + /** The child's open claims its outcome; false if the child no longer starts on this clock. */ + own(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): boolean { + if (!this.tracking(sessionId, child)) { + return false + } + this.owners.set(sessionId, { generation: child.generation, fence: child.fence }) + return true + } + + disown(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): void { + if (this.ownsOpen(sessionId, child)) { + this.owners.delete(sessionId) + } + } + + ownsOpen(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): boolean { + const owner = this.owners.get(sessionId) + return !!owner && sameProviderChild(owner, child) + } + + /** The owner's verdict; readiness waiters and the start flight read it once readiness ends. */ + failed( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + failure: AgentSessionStartFailure + ): void { + const tracked = this.tracking(sessionId, child) + if (tracked) { + tracked.failure = failure + this.settle(sessionId, tracked, 'ended') + } + } + /** Published starts retain their clock; expire immediately if acquisition outlasted its limit. */ published( sessionId: string, @@ -199,13 +172,8 @@ export class StructuredAgentSessionStartupAttempts { /** Reject readiness that lost the race to expiry. */ onClock(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): boolean { - const tracked = this.open.get(sessionId) - return ( - !this.disposed && - !!tracked?.child && - !tracked.expired && - sameProviderChild(tracked.child, child) - ) + const tracked = this.tracking(sessionId, child) + return !this.disposed && !!tracked && !tracked.expired } ready(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): void { @@ -215,9 +183,34 @@ export class StructuredAgentSessionStartupAttempts { } } - childEnded(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): void { - const tracked = this.open.get(sessionId) - if (tracked?.child && sameProviderChild(tracked.child, child)) { + /** Settles once the adapter saw this starting child's own root exit; otherwise never. */ + rootExit(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): Promise { + return this.tracking(sessionId, child)?.rootExit.promise ?? new Promise(() => {}) + } + + /** Orca ending the child decides the start: a failure Orca recorded is final, any other close + * cancels it. An exit of its own leaves an open still settling the start, and its clock, until + * that open gives the outcome. */ + childEnded( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity, + ended: { byOrca: boolean; failure?: SubmissionRejectionFact; reason?: string | null } = { + byOrca: false + } + ): void { + const tracked = this.tracking(sessionId, child) + if (ended.failure) { + const cause = new Error(ended.reason ?? 'Orca stopped the agent while it started') + this.failed(sessionId, child, new AgentSessionStartFailure(cause, 'never', ended.failure)) + } else if (tracked && (ended.byOrca || !this.ownsOpen(sessionId, child))) { + this.settle(sessionId, tracked, 'ended') + } + } + + /** The owner ended without a verdict: replaced, cancelled or no longer starting. */ + released(sessionId: string, child: StructuredAgentSessionProviderChildIdentity): void { + const tracked = this.tracking(sessionId, child) + if (tracked) { this.settle(sessionId, tracked, 'ended') } } @@ -238,11 +231,34 @@ export class StructuredAgentSessionStartupAttempts { this.disposed = true for (const tracked of this.open.values()) { tracked.readiness.resolve('ended') + tracked.stopListening?.() if (tracked.timer) { clearTimeout(tracked.timer) } } this.open.clear() + this.owners.clear() + } + + private tracking( + sessionId: string, + child: StructuredAgentSessionProviderChildIdentity + ): Tracked | undefined { + const tracked = this.open.get(sessionId) + return tracked?.child && sameProviderChild(tracked.child, child) ? tracked : undefined + } + + /** The deadline becomes the earlier of the original one and the grace after the exit. */ + private rootExited(sessionId: string, tracked: Tracked): void { + if (this.open.get(sessionId) !== tracked || this.disposed || tracked.rootExitAt !== null) { + return + } + tracked.rootExitAt = this.now() + tracked.rootExit.resolve() + if (tracked.timer) { + clearTimeout(tracked.timer) + } + this.schedule(sessionId, tracked) } private markSpawned(tracked: Tracked): void { @@ -250,7 +266,7 @@ export class StructuredAgentSessionStartupAttempts { } private schedule(sessionId: string, tracked: Tracked): void { - const due = tracked.registeredAt + this.limits.ceilingMs + const due = startupDue(tracked, this.limits.ceilingMs) tracked.timer = setTimeout(() => this.check(sessionId, tracked), Math.max(0, due - this.now())) tracked.timer.unref?.() } @@ -262,11 +278,25 @@ export class StructuredAgentSessionStartupAttempts { } if (this.now() - tracked.registeredAt >= this.limits.ceilingMs) { this.expire(sessionId, tracked, 'ceiling') + } else if ( + tracked.rootExitAt !== null && + this.now() - tracked.rootExitAt >= STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS + ) { + this.breach(sessionId, tracked) } else { this.schedule(sessionId, tracked) } } + /** Fires on a timer, so neither the adapter nor the session's serialized work can hold it. */ + private breach(sessionId: string, tracked: Tracked): void { + const agent = tracked.attempt?.identity.agent ?? 'agent' + tracked.failure = startupContractFailure(agent) + const breached = { sessionId, attemptId: tracked.attemptId, child: tracked.child, agent } + this.settle(sessionId, tracked, 'ended') + this.deps.breach?.(breached) + } + /** null means an already-expired spawn just published its child. */ private expire(sessionId: string, tracked: Tracked, reason: 'ceiling' | null): void { if (this.open.get(sessionId) !== tracked || this.disposed) { @@ -296,13 +326,7 @@ export class StructuredAgentSessionStartupAttempts { tracked: Tracked, outcome: StructuredAgentSessionStartupSettled['outcome'] ): StructuredAgentSessionStartupSettled | null { - return tracked.spawnedAt === null || !tracked.attempt - ? null - : { - agent: tracked.attempt.identity.agent, - outcome, - durationMs: Math.max(0, this.now() - tracked.spawnedAt) - } + return measureStartup(tracked, outcome, this.now()) } private report(settled: StructuredAgentSessionStartupSettled | null): void { @@ -319,6 +343,7 @@ export class StructuredAgentSessionStartupAttempts { private end(sessionId: string): void { const tracked = this.open.get(sessionId) tracked?.readiness.resolve('ended') + tracked?.stopListening?.() if (tracked?.timer) { clearTimeout(tracked.timer) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts new file mode 100644 index 000000000000..f0e5d3881986 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-liveness.test.ts @@ -0,0 +1,205 @@ +import { PassThrough } from 'node:stream' +import { describe, expect, it, vi } from 'vitest' +import { createAcpAgentConnection } from '../../acp/acp-agent-connection' +import { openAcpAdapterRig } from '../../acp/acp-structured-adapter.test-support' +import { readAntigravitySignInLine } from '../../antigravity/antigravity-sign-in-url' +import { CodexAppServerExitError } from '../../codex/app-server/codex-app-server-exit-error' +import { adapterFor, fakeCodex } from '../../codex/codex-structured-session-adapter-fixture' +import { createFakePipedChild } from '../../../shared/__fixtures__/fake-spawned-child' +import { closeProviderTimelineRigs } from '../agent-session-timeline/provider-timeline-assembler-test-support' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { findInErrorChain } from '../../provider-process/provider-error-chain' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' +import { recordStartedHandle } from './structured-agent-session-start-provider-answer' +import type * as ProviderAnswer from './structured-agent-session-start-provider-answer' +import { STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS } from './structured-agent-session-startup-attempt-contract' +import { + accept, + acquire, + dispatch, + eventually, + heldBehindStart, + host, + observeRootExit, + proveOpen, + providerOpen, + publishFirst, + recordStartupEnding, + registerStartupAttemptHostTests, + restartWith, + submission +} from './structured-agent-session-startup-attempt-host-test-support' + +vi.mock('./structured-agent-session-start-provider-answer', async (original) => { + const actual = await original() + return { ...actual, recordStartedHandle: vi.fn(actual.recordStartedHandle) } +}) + +registerStartupAttemptHostTests() + +/** Holds the session's serialized work, as a stalled journal write would. */ +async function holdSerializedWork(): Promise<() => Promise> { + const held = Promise.withResolvers() + const entered = Promise.withResolvers() + const blocked = host.collaboratorsForTests().serialize(SESSION, async () => { + entered.resolve() + await held.promise + }) + await entered.promise + return async () => { + held.resolve() + await blocked + } +} + +describe('a start that cannot finish', () => { + it.each([ + ['its process exited and the adapter never answered', 'providerStartFailed', 0], + ['it outlived its ceiling', 'expired', 20_000] + ] as const)( + 'releases its callers when %s, while the session is busy, and records its row once after', + async (_case, decided, waitMs) => { + await restartWith(publishFirst, { startupLimits: { ceilingMs: 20_000 } }) + providerOpen.mockImplementationOnce(() => new Promise(() => {})) + const id = await heldBehindStart('one request') + const flight = host.collaboratorsForTests().runtimeState.startFlights.get(SESSION)! + const release = await holdSerializedWork() + let outcome: unknown + void flight.outcome.then((settled) => { + outcome = settled + }) + if (waitMs === 0) { + observeRootExit() + } + await vi.advanceTimersByTimeAsync(waitMs || STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS) + expect(outcome).toMatchObject( + decided === 'expired' + ? { ok: false, expired: true } + : { ok: false, failure: { kind: 'providerStartFailed' } } + ) + await release() + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + const rows = (await host.journalSnapshot(SESSION)).items.filter( + (item) => item.body.kind === 'status' + ) + expect(rows).toHaveLength(1) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + } + ) + + it("wakes on Codex's own exit while spawn bookkeeping is still held, and never readies after", async () => { + const codex = fakeCodex() + const identity = Promise.withResolvers() + const adapter = adapterFor(codex, {}, [], { readProcessStartTime: () => identity.promise }) + await restartWith((input) => adapter.acquire(input)) + const id = await accept('a start whose process dies before its identity is saved') + await eventually(() => expect(codex.connections).toHaveLength(1)) + const child = codex.connections[0] + child.closed = true + child.handlers.onExit?.(new CodexAppServerExitError('codex app-server connection ended'), { + expected: false + }) + await vi.advanceTimersByTimeAsync(STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS) + expect(host.collaboratorsForTests().runtimeState.startupAttempts.isOpen(SESSION)).toBe(false) + expect(acquire.mock.calls[0][0].signal?.aborted).toBe(true) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + identity.resolve(1_700_000_000_000) + await vi.advanceTimersByTimeAsync(1_000) + expect(host.readStatusSummary(SESSION)?.hostExecutionPhase).not.toBe('ready') + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + await adapter.closeAll() + }) +}) + +describe("Orca's own refusal during a start", () => { + it('stays final when its handle write refuses after the agent died unanswered', async () => { + await restartWith(publishFirst) + const write = Promise.withResolvers() + const entered = Promise.withResolvers() + vi.mocked(recordStartedHandle).mockImplementationOnce(() => { + entered.resolve() + return write.promise + }) + const id = await heldBehindStart('the thread opened; saving its handle is still pending') + await proveOpen() + await entered.promise + recordStartupEnding() + observeRootExit() + await vi.advanceTimersByTimeAsync(10) + write.resolve(false) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect((await submission(id))?.rejection).toMatchObject({ kind: 'hostStopped' }) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('stays final when an ACP agent prints a sign-in line Orca cannot read before it initializes', async () => { + const connections: ReturnType[] = [] + const children: ReturnType[] = [] + let refusal: Error | undefined + let acquireError: unknown + const rig = await openAcpAdapterRig({ + deps: { + connect: (launch, options) => { + const child = Object.assign(createFakePipedChild(), { + pid: 9_999_999, + stdout: new PassThrough(), + stdin: new PassThrough(), + stderr: new PassThrough(), + kill: () => true + }) + children.push(child) + // Orca's cleanup ends stdin; the agent then exits. + child.stdin.once('finish', () => child.emit('exit', 0, null)) + const connection = createAcpAgentConnection( + launch, + { + ...options, + onClose: (error) => { + refusal ??= error + options.onClose?.(error) + }, + signIn: { read: readAntigravitySignInLine } + }, + () => child + ) + connections.push(connection) + return connection + }, + // The refusal and its cleanup exit land while the spawn's identity is still being saved. + readProcessStartTime: async () => { + children + .at(-1)! + .stdout.write('Open the following link to authenticate the ACP server: not-a-url\n') + await eventually(() => expect(connections.at(-1)!.exited).toBe(true)) + return 1_700_000_000_000 + } + } + }) + try { + await restartWith((input) => + rig.adapter.acquire(input).catch((error: unknown) => { + acquireError = error + throw error + }) + ) + const id = await accept('an agent whose sign-in line Orca cannot read') + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + // The adapter's own final verdict carries Orca's original refusal, not a generic close. + expect(refusal?.message).toContain('sign-in link Orca cannot read') + const verdict = findInErrorChain(acquireError, (cause) => + cause instanceof AgentSessionStartFailure ? cause : undefined + ) + expect(verdict?.retry).toBe('never') + expect(verdict?.cause).toBe(refusal) + expect(connections).toHaveLength(1) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() + } finally { + await rig.adapter.closeAll() + await closeProviderTimelineRigs() + } + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-provider-errors.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-provider-errors.test.ts index 751c46678b8b..b100e4a05ef1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-provider-errors.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-provider-errors.test.ts @@ -208,19 +208,21 @@ it.each(['result', 'stderr exit', 'result then exit', 'stderr exit then result'] ) // Real order: the result lands before the exit, so its own error decides the attempt. +// Claude's answer is final; only an exit with no answer retries. it.each([ - ['result', 'startFailed', "Claude couldn't start. Send your message to try again."], - ['result then exit', 'startFailed', "Claude couldn't start. Send your message to try again."], + ['result', 1, 'startFailed', "Claude couldn't start. Send your message to try again."], + ['result then exit', 1, 'startFailed', "Claude couldn't start. Send your message to try again."], [ 'stderr exit', + 5, 'providerStartFailed', 'Claude stopped before it finished starting. Send your message to try again.' ] ] as const)( - 'retries a generic startup failure (%s) up to five times, then shows one row', - async (order, kind, text) => { + 'ends a generic startup failure (%s) after %i start(s) with one row', + async (order, starts, kind, text) => { const id = await send('continue this chat') - for (let attempt = 1; attempt <= 5; attempt += 1) { + for (let attempt = 1; attempt <= starts; attempt += 1) { const connection = await starting(attempt) if (order !== 'stderr exit') { frame(connection, GENERIC) @@ -234,7 +236,7 @@ it.each([ } } const errors = await settled(id) - expect(claude.connections).toHaveLength(5) + expect(claude.connections).toHaveLength(starts) if (order === 'result') { // The host stops each failed child; the result itself is no exit evidence. for (const connection of claude.connections) { @@ -247,6 +249,17 @@ it.each([ text, failure: { kind, detail: { text: expect.stringContaining(GENERIC_TEXT), audience: 'log' } } }) + if (order !== 'stderr exit') { + // The ended start logs what Claude said. + expect(log.entries).toContainEqual( + expect.objectContaining({ + message: 'a provider could not finish starting', + fields: expect.objectContaining({ + error: expect.objectContaining({ message: expect.stringContaining(GENERIC_TEXT) }) + }) + }) + ) + } } ) @@ -271,9 +284,9 @@ it('leaves a Stop during startup to Stop when Claude then reports a startup fail expect(claude.connections).toHaveLength(1) }) -it('recovers from a generic startup failure with no failure row, logging what Claude said', async () => { +it('recovers from an unanswered startup exit with no failure row', async () => { await send('continue this chat') - frame(await starting(1), GENERIC) + exit(await starting(1), GENERIC_TEXT) const recovered = await starting(2) recovered.initialization.resolve({}) // The queued send reaches the recovered Claude. @@ -283,12 +296,4 @@ it('recovers from a generic startup failure with no failure row, logging what Cl expect(items.filter((item) => item.body.kind === 'status' && item.body.tone === 'error')).toEqual( [] ) - expect(log.entries).toContainEqual( - expect.objectContaining({ - message: 'a provider could not finish starting', - fields: expect.objectContaining({ - error: expect.objectContaining({ message: expect.stringContaining(GENERIC_TEXT) }) - }) - }) - ) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-tracking.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-tracking.ts new file mode 100644 index 000000000000..39ac73dc74cd --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-tracking.ts @@ -0,0 +1,98 @@ +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session/journal/agent-session-journal-identity' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' +import type { StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' +import { + STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS, + StructuredAgentSessionStartupContractError, + type StructuredAgentSessionStartupAttempt +} from './structured-agent-session-startup-attempt-contract' +import { AgentSessionStartFailure } from './structured-agent-session-start-failure' + +export type StructuredAgentSessionExpiredStartup = { + sessionId: string + attemptId: string + child: StructuredAgentSessionProviderChildIdentity | null +} + +export type StructuredAgentSessionStartupSettled = { + agent: AgentSessionJournalIdentity['agent'] + outcome: 'ready' | 'ceiling' | 'ended' + durationMs: number +} + +export type StructuredAgentSessionStartupProgress = { + spawned: () => void +} + +export type StructuredAgentSessionTrackedStartup = { + attemptId: string + attempt: StructuredAgentSessionStartupAttempt | null + child: StructuredAgentSessionProviderChildIdentity | null + /** Absolute ceiling starts before any startup wait. */ + registeredAt: number + spawnedAt: number | null + timer: ReturnType | null + expired: boolean + /** Defer expiry reporting until aborted spawn settles; successful readiness overrides it. */ + expiry: StructuredAgentSessionStartupSettled | null + readiness: PromiseWithResolvers<'ready' | 'ended'> + /** When the adapter first saw this acquisition's own root exit; it shortens the deadline. */ + rootExitAt: number | null + rootExit: PromiseWithResolvers + stopListening?: () => void + /** Set before readiness settles when the start's owner or the contract guard failed it. */ + failure?: AgentSessionStartFailure +} + +export function trackedStartup( + attemptId: string, + registeredAt: number +): StructuredAgentSessionTrackedStartup { + return { + attemptId, + attempt: null, + child: null, + registeredAt, + spawnedAt: null, + timer: null, + expired: false, + expiry: null, + readiness: Promise.withResolvers(), + rootExitAt: null, + rootExit: Promise.withResolvers() + } +} + +/** The earlier of the original deadline and the grace after the acquisition's own root exit. */ +export function startupDue( + tracked: StructuredAgentSessionTrackedStartup, + ceilingMs: number +): number { + return Math.min( + tracked.registeredAt + ceilingMs, + (tracked.rootExitAt ?? Infinity) + STRUCTURED_AGENT_SESSION_STARTUP_EXIT_GRACE_MS + ) +} + +/** A broken adapter contract fails its start for good, worded as the agent failing to start. */ +export function startupContractFailure(agent: string): AgentSessionStartFailure { + return new AgentSessionStartFailure( + new StructuredAgentSessionStartupContractError(agent), + 'never', + agentSessionFailureFact('providerStartFailed') + ) +} + +export function measureStartup( + tracked: StructuredAgentSessionTrackedStartup, + outcome: StructuredAgentSessionStartupSettled['outcome'], + now: number +): StructuredAgentSessionStartupSettled | null { + return tracked.spawnedAt === null || !tracked.attempt + ? null + : { + agent: tracked.attempt.identity.agent, + outcome, + durationMs: Math.max(0, now - tracked.spawnedAt) + } +} diff --git a/src/main/opencode/opencode-acp-connection.ts b/src/main/opencode/opencode-acp-connection.ts index de4e6a64b7b9..a38d6d942aff 100644 --- a/src/main/opencode/opencode-acp-connection.ts +++ b/src/main/opencode/opencode-acp-connection.ts @@ -109,7 +109,8 @@ class OpenCodeAcpConnection extends AcpAgentConnection { } catch { // Native history is best effort; never hide the ACP failure when it cannot be read. } - if (!failure || this.closed || failure.inputId === this.lastRecoveredInput) { + // An exit while reading ends the connection even while its output still drains. + if (!failure || this.closed || this.exited || failure.inputId === this.lastRecoveredInput) { throw error } this.lastRecoveredInput = failure.inputId diff --git a/src/main/pi/pi-scripted-child.test-fixture.ts b/src/main/pi/pi-scripted-child.test-fixture.ts index 12e72e70b3b3..7eb8c23634a5 100644 --- a/src/main/pi/pi-scripted-child.test-fixture.ts +++ b/src/main/pi/pi-scripted-child.test-fixture.ts @@ -4,7 +4,10 @@ import { providerDiagnostic, withProviderDiagnostic } from '../../shared/agent-session/failures/agent-session-failure' -import type { JsonlRpcAgentConnectionOptions } from '../jsonl-rpc/agent-connection' +import { + JsonlRpcAgentExitError, + type JsonlRpcAgentConnectionOptions +} from '../jsonl-rpc/agent-connection' import { JsonlRpcResponseError, type JsonlRpcRecord } from '../jsonl-rpc/peer' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' import type { ScriptedAgentChild } from '../runtime/structured-agent-scripted-child.test-fixture' @@ -156,6 +159,16 @@ class ScriptedPiConnection implements PiRpcConnection { this.exit(error) } + exitHandshake(message: string): void { + // As the real connection: an exit fails the pending calls with its own exit error. + const error = withProviderDiagnostic( + new JsonlRpcAgentExitError(message), + providerDiagnostic(message, 'log') + ) + this.handshake.reject(error) + this.exit(error) + } + async close(): Promise>> { if (!this.closed) { this.onClosed() @@ -288,6 +301,7 @@ export const piScriptedChild = (options: PiScriptOptions = {}): ScriptedPiChild }, releaseHandshake: () => children.at(-1)?.releaseHandshake(), failHandshake: (message) => children.at(-1)?.failHandshake(message), + exitHandshake: (message) => children.at(-1)?.exitHandshake(message), prompts: () => received, spawns: () => children.length, resumes: () => resumed, diff --git a/src/main/pi/rpc-session-adapter.ts b/src/main/pi/rpc-session-adapter.ts index bffaf70d0a40..c55f9f5dbac8 100644 --- a/src/main/pi/rpc-session-adapter.ts +++ b/src/main/pi/rpc-session-adapter.ts @@ -18,6 +18,9 @@ import { buildPiRpcLaunch } from './rpc-launch' import type { PiRpcResolvedLaunch } from './rpc-launch-resolution' import { PiRpcSession, type PiRpcSessionDeps, type PiRpcConnection } from './rpc-session' import { dispatchPiRpcMessage } from './rpc-message-dispatch' +import { piAcquireFailure } from './rpc-startup-outcome' +import { openPiStart, watchPiStartExit } from './rpc-startup-open' +import { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' import { applyPiRpcSessionOption } from './rpc-options' import { supportsSupervisedProviderChildLocation } from '../provider-process/supervised-provider-child-location' @@ -33,6 +36,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { private readonly acquiring = new Set() private readonly starts = new ProviderAcquisitionStarts() private readonly retiring = new Set>() + private readonly endings = new AgentSessionStartEndings() constructor(private readonly deps: PiRpcSessionAdapterDeps) {} supportsLocation = supportsSupervisedProviderChildLocation @@ -81,6 +85,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { const fresh = !launch.sessionFile && !launch.forkFile session = new PiRpcSession(input, randomUUID(), spec, this.deps, launch, fresh) this.sessions.set(id, session) + watchPiStartExit(session, input, this.endings) this.starts.track(attempt, session.connection) const spawned = providerSpawnedProcessIdentity( input, @@ -105,10 +110,11 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { ? error : new AgentSessionPreSpawnError(error) } + this.endings.retire(id, session.generation) const result = await session.close(false).catch(() => null) const exited = piStartExitError(error, session.connection, result) if (exited) { - throw exited + throw piAcquireFailure(exited) } this.starts.retainFailed(id, session.connection) throw new AgentSessionAcquisitionExitUnprovenError(error) @@ -118,18 +124,8 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { } } - open: NonNullable = async (input) => { - const session = this.sessions.get(input.sessionId) - if ( - !session || - session.input.fence !== input.fence || - session.generation !== input.acquisitionGeneration || - session.connection.closed - ) { - throw new Error('Pi child changed before it opened') - } - return session.open(input.signal) - } + open: NonNullable = (input) => + openPiStart(this.sessions, this.endings, input) dispatch: StructuredAgentSessionAdapter['dispatch'] = async (input) => { const session = this.session(input.sessionId, input.fence) @@ -232,12 +228,16 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { } holdsDispatch = (id: string): boolean => this.sessions.get(id)?.turns.holdsDispatch ?? false heldSends = (id: string): ReadonlySet | undefined => this.sessions.get(id)?.heldSends + startupEnding = (id: string, generation: string) => this.endings.get(id, generation) + retireStartup = (id: string, generation: string) => this.endings.retire(id, generation) holdsLiveProviderProcess(id: string, generation: string): boolean { const session = this.sessions.get(id) return session?.generation === generation && !session.connection.closed } closeSession(id: string): Promise { + // A released or closed session's start has no reader left for its ending. + this.endings.retire(id) return this.close(id, true) } private async close(id: string, requested: boolean): Promise { @@ -283,6 +283,7 @@ export class PiRpcSessionAdapter implements StructuredAgentSessionAdapter { void retirement.then(() => this.retiring.delete(retirement)) } async closeAll(): Promise { + this.endings.clear() await Promise.all( [...this.sessions.keys()].map((id) => stopAgentSessionProviderRoot(() => this.closeSession(id)) diff --git a/src/main/pi/rpc-session-startup.test.ts b/src/main/pi/rpc-session-startup.test.ts index 438671c19e6a..a68b81f7b70c 100644 --- a/src/main/pi/rpc-session-startup.test.ts +++ b/src/main/pi/rpc-session-startup.test.ts @@ -93,7 +93,8 @@ describe('Pi host awaited open', () => { command === 'get_state' ? Promise.resolve(unknown) : undefined }) await expect(openAt8(h, acquired.acquisitionGeneration)).rejects.toMatchObject({ - reason: 'notSignedIn' + retry: 'never', + cause: { reason: 'notSignedIn' } }) }) diff --git a/src/main/pi/rpc-session.ts b/src/main/pi/rpc-session.ts index 9aae0b4b97d5..0938316191d0 100644 --- a/src/main/pi/rpc-session.ts +++ b/src/main/pi/rpc-session.ts @@ -66,14 +66,15 @@ export class PiRpcSession { } private optionObservations = new PiRpcOptionObservations( this.optionProof, - () => !this.requestedClose && !this.failedCause && !this.connection.closed + () => !this.requestedClose && !this.failure && !this.connection.closed ) readonly startup: PiRpcSessionStartup readonly history = PiHistoryCapture.forSession(this, () => this.deps) openedLink: AgentSessionProviderHandleLink | null = null requestedClose = false private publishedExit = false - private failedCause?: Error + /** What ended this child, once something did. */ + failure?: Error private exitDelivery: Promise = Promise.resolve() private releaseAfterExit?: () => void private readonly reading = new ReaderPauseHolds<'lane' | 'journal'>(() => this.connection) @@ -163,7 +164,7 @@ export class PiRpcSession { this.lane.apply(this.turns.context.setModel(state.model ?? undefined, Date.now())) this.optionObservations.seed(state) }, - () => this.failedCause, + () => this.failure, () => this.optionObservations.prepare() ) } @@ -211,7 +212,7 @@ export class PiRpcSession { } fail(error: Error): void { - this.failedCause ??= error + this.failure ??= error this.deps.logger.error('Pi RPC session failed', { scope: 'pi-rpc', sessionId: this.input.identity.sessionId, @@ -264,7 +265,7 @@ export class PiRpcSession { this.publishedExit = true const observedAt = Date.now() this.input.events?.observeExit?.(this.heldSends, observedAt) - const cause = (this.failedCause ??= error) + const cause = (this.failure ??= error) this.dialogs.cancelAll() this.turns.end() // The lane is sealed once finalized, so the end event travels with the final transfer. diff --git a/src/main/pi/rpc-startup-open.ts b/src/main/pi/rpc-startup-open.ts new file mode 100644 index 000000000000..2be19aa7c091 --- /dev/null +++ b/src/main/pi/rpc-startup-open.ts @@ -0,0 +1,66 @@ +import type { + StructuredAgentSessionAcquireInput, + StructuredAgentSessionAdapter +} from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { AgentSessionStartEndings } from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import type { StructuredAgentSessionProviderOpenInput } from '../native-chat/agent-session-wire/structured-agent-session-provider-open-contract' +import { JsonlRpcAgentExitError } from '../jsonl-rpc/agent-connection' +import { withProviderSpawnAbsence } from '../provider-process/provider-executable-missing' +import type { PiRpcSession } from './rpc-session' +import { piStartFailure } from './rpc-startup-outcome' + +/** Takes this child's own exit when first seen, so a start settling meanwhile cannot miss it; the + * verdict weighs a failure before it against that exit. */ +export function watchPiStartExit( + session: PiRpcSession, + input: Pick, + endings: AgentSessionStartEndings +): void { + const sessionId = session.input.identity.sessionId + endings.admit(sessionId, session.generation) + session.connection.onExit((requested) => { + if (!requested) { + input.events?.observeRootExit?.() + const exited = withProviderSpawnAbsence( + new JsonlRpcAgentExitError('Pi exited while starting'), + session.connection.spawnAbsence + ) + endings.record(sessionId, session.generation, piStartFailure([session.failure, exited])) + } + }) +} + +/** A start's open answers with Pi's verdict; a child that already ended answers with how it + * ended, and an answer its own exit followed is still worded as that exit. */ +export async function openPiStart( + sessions: Map, + endings: AgentSessionStartEndings, + input: StructuredAgentSessionProviderOpenInput +): ReturnType> { + const session = sessions.get(input.sessionId) + try { + if ( + !session || + session.input.fence !== input.fence || + session.generation !== input.acquisitionGeneration + ) { + throw ( + endings.get(input.sessionId, input.acquisitionGeneration) ?? + new Error('Pi child changed before it opened') + ) + } + if (session.connection.closed) { + throw ( + endings.get(input.sessionId, input.acquisitionGeneration) ?? + session.failure ?? + new Error('Pi child ended before it opened') + ) + } + return await session.open(input.signal) + } catch (error) { + throw piStartFailure( + error, + session?.connection.rootVerdict === 'exited' && !session.requestedClose + ) + } +} diff --git a/src/main/pi/rpc-startup-outcome.ts b/src/main/pi/rpc-startup-outcome.ts new file mode 100644 index 000000000000..47313f701f0f --- /dev/null +++ b/src/main/pi/rpc-startup-outcome.ts @@ -0,0 +1,25 @@ +import { + agentSessionAcquireFailure, + agentSessionStartFailure, + type AgentSessionStartAnswer, + type AgentSessionStartFailure +} from '../native-chat/agent-session-wire/structured-agent-session-start-failure' +import { JsonlRpcAgentExitError } from '../jsonl-rpc/agent-connection' +import { JsonlRpcResponseError } from '../jsonl-rpc/peer' + +/** Pi declares no temporary startup answers. Only its process's own exit, once its output + * drained, leaves a start unanswered; a closed stream alone proves no exit. */ +export const piStartupAnswer: AgentSessionStartAnswer = (cause) => { + if (cause instanceof JsonlRpcResponseError) { + return 'final' + } + return cause instanceof JsonlRpcAgentExitError ? 'exited' : undefined +} + +export function piStartFailure(error: unknown, exited = false): AgentSessionStartFailure { + return agentSessionStartFailure(error, piStartupAnswer, exited) +} + +export function piAcquireFailure(error: unknown): unknown { + return agentSessionAcquireFailure(error, piStartupAnswer) +} diff --git a/src/main/provider-process/provider-exit-drain-scaling.test.ts b/src/main/provider-process/provider-exit-drain-scaling.test.ts new file mode 100644 index 000000000000..4d1014224744 --- /dev/null +++ b/src/main/provider-process/provider-exit-drain-scaling.test.ts @@ -0,0 +1,293 @@ +import type { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' +import { afterEach, expect, it, vi } from 'vitest' +import type { spawnProcess } from '@orca/process-host' +import type { PipedProcessSpawner } from '@orca/process-host/process-spec' +import { createFakePipedChild } from '../../shared/__fixtures__/fake-spawned-child' +import { createAcpAgentConnection } from '../acp/acp-agent-connection' +import { AcpScriptedAgent, type FakeFrame } from '../acp/acp-scripted-agent.test-support' +import { openCodexAppServerConnection } from '../codex/app-server/codex-app-server-connection' +import { JsonlRpcAgentConnection } from '../jsonl-rpc/agent-connection' +import { PROVIDER_EXIT_OUTPUT_DRAIN_MS } from './provider-exit-output-drain' +import type * as OutputDrain from './provider-exit-output-drain' +import type { terminateProviderProcessTree } from './provider-process-teardown' + +// Every output drain any transport starts, the listeners it attached, and whether it finished. +type Attached = { emitter: EventEmitter; listener: ReturnType[number] } +const drains = vi.hoisted(() => new Array<{ attached: Attached[]; finished: number }>()) +vi.mock('./provider-exit-output-drain', async (original) => { + const actual = await original() + const watched = (stdout: EventEmitter, child?: EventEmitter): Attached[] => + [ + { emitter: stdout, event: 'end' }, + { emitter: stdout, event: 'close' }, + ...(child ? [{ emitter: child, event: 'close' }] : []) + ].flatMap(({ emitter, event }) => + emitter.listeners(event).map((listener) => ({ emitter, listener })) + ) + return { + ...actual, + afterProviderOutputDrains: ( + ...[stdout, finish, child]: Parameters + ) => { + const before = new Set(watched(stdout, child).map(({ listener }) => listener)) + const drain: { attached: Attached[]; finished: number } = { attached: [], finished: 0 } + drains.push(drain) + const done = actual.afterProviderOutputDrains( + stdout, + () => { + drain.finished += 1 + finish() + }, + child + ) + drain.attached = watched(stdout, child).filter(({ listener }) => !before.has(listener)) + return done + } + } +}) +vi.mock('./provider-process-teardown', () => ({ + terminateProviderProcessTree: vi.fn( + async () => 'unverifiable' + ) +})) + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +type Child = ReturnType +/** One connection over a fake child, with a call the agent has not answered yet. */ +type Opened = { + child: Child + pending: Promise + /** The agent's own answer to that call, written to its stdout. */ + answer: () => void + close: () => Promise + lostBeforeExit?: () => Error | undefined +} + +function fakeChild(stdout = new PassThrough(), stdin = new PassThrough()) { + return Object.assign(createFakePipedChild(), { + // Outside any real process table so teardown never mistakes an unrelated process for it. + pid: 9_999_999, + stdin, + stdout, + stderr: new PassThrough(), + kill: () => true + }) +} + +const TRANSPORTS: Record Promise> = { + acp: async () => { + const agent = new AcpScriptedAgent() + const child = fakeChild(agent.stdout, agent.stdin) + let prompt: FakeFrame | undefined + agent.on('initialize', (frame) => agent.reply(frame, { protocolVersion: 1 })) + agent.on('session/new', (frame) => agent.reply(frame, { sessionId: 'session-1' })) + agent.on('session/prompt', (frame) => { + prompt = frame + }) + const spawn: PipedProcessSpawner = () => child + const connection = createAcpAgentConnection( + { command: 'fixture-acp-agent', args: ['--acp'], cwd: '/execution-host/folder', env: {} }, + {}, + spawn + ) + await connection.start({ cwd: '/execution-host/folder', mcpServers: [] }) + const pending = connection.prompt([{ type: 'text', text: 'hello' }]) + await vi.waitFor(() => expect(prompt).toBeDefined()) + return { + child, + pending, + answer: () => agent.reply(prompt!, { stopReason: 'end_turn' }), + close: () => connection.close(), + lostBeforeExit: () => connection.lostBeforeExit + } + }, + jsonl: async () => { + const child = fakeChild() + const spawn: PipedProcessSpawner = () => child + const connection = new JsonlRpcAgentConnection( + { command: 'fixture-rpc-agent', args: ['--mode', 'rpc'], cwd: '/folder', env: {} }, + {}, + spawn + ) + return { + child, + pending: connection.request('get_state'), + answer: () => + child.stdout.write( + '{"type":"response","command":"get_state","id":"orca-1","success":true,"data":{}}\n' + ), + close: () => connection.close() + } + }, + codex: async () => { + const child = fakeChild() + child.stdin.once('data', () => child.stdout.write(`${JSON.stringify({ id: 1, result: {} })}\n`)) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The connection reads only events, pid, streams and kill from this stub. + const spawnImpl = (() => child) as unknown as typeof spawnProcess + const connection = await openCodexAppServerConnection( + { command: 'codex', args: ['app-server'] }, + {}, + spawnImpl + ) + return { + child, + pending: connection.request('thread/start', {}), + answer: () => child.stdout.write(`${JSON.stringify({ id: 2, result: {} })}\n`), + close: () => connection.close() + } + } +} + +/** Listeners an output drain attached that are still attached. */ +const drainListeners = () => + drains.reduce( + (sum, drain) => + sum + + drain.attached.filter(({ emitter, listener }) => + [...emitter.listeners('end'), ...emitter.listeners('close')].includes(listener) + ).length, + 0 + ) + +/** Runs `count` connections through each terminal order and counts what each one leaves. */ +async function drainCycles(open: () => Promise, count: number) { + drains.length = 0 + const timers = vi.spyOn(globalThis, 'setTimeout') + const settled = { answered: 0, failed: 0 } + const left: { timers: number; drainListeners: number }[] = [] + for (let cycle = 0; cycle < count; cycle++) { + for (const answered of [true, false]) { + const opened = await open() + const { child } = opened + opened.pending.then( + () => (settled.answered += 1), + () => (settled.failed += 1) + ) + if (answered) { + // The exit is seen before the answer written just before it; the drain waits for it. + child.emit('exit', 0, null) + child.emit('exit', 0, null) + opened.answer() + child.stdout.end() + } else { + // The agent stops reading, twice over, and exits with its stdout still held open. + const lost = new Error('broken pipe') + child.stdin.emit('error', lost) + child.stdin.emit('error', new Error('a later failure')) + child.stdin.emit('close') + child.emit('exit', 1, null) + child.emit('exit', 1, null) + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS) + // ACP keeps why the agent stopped reading, its first reason only, for its exit's readers. + expect(opened.lostBeforeExit?.() ?? lost).toBe(lost) + } + await vi.advanceTimersByTimeAsync(0) + // Read before the child's own close, which would fire any listener a drain left behind. + const listening = drainListeners() + await opened.close() + child.emit('close', 0, null) + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS) + left.push({ timers: vi.getTimerCount(), drainListeners: listening }) + } + } + const timersScheduled = timers.mock.calls.length + timers.mockRestore() + const finished = drains.map((drain) => drain.finished) + return { settled, left, timersScheduled, drains: drains.length, finished } +} + +it.each(Object.keys(TRANSPORTS))( + '%s: settles each pending call once and leaves no drain timer or listener, at N and 10N', + async (transport) => { + vi.useFakeTimers() + const n = await drainCycles(TRANSPORTS[transport], 10) + const tenN = await drainCycles(TRANSPORTS[transport], 100) + for (const run of [n, tenN]) { + for (const held of run.left) { + expect(held).toEqual({ timers: 0, drainListeners: 0 }) + } + } + // An answer written before exit wins; an unanswered call fails once, at the drain bound. + expect(n.settled).toEqual({ answered: 10, failed: 10 }) + expect(tenN.settled).toEqual({ answered: 100, failed: 100 }) + expect(tenN.timersScheduled).toBe(10 * n.timersScheduled) + // Each drain finishes exactly once, and a connection starts a fixed number of them. + for (const run of [n, tenN]) { + expect(run.finished.every((finished) => finished === 1)).toBe(true) + } + expect(n.drains).toBeGreaterThan(0) + expect(tenN.drains).toBe(10 * n.drains) + } +) + +/** JSON-lines output ending while its root still runs, then the exit after it, no exit at all, or + * Orca closing while an exit's drain is pending; counts what each cycle leaves at its close. */ +async function jsonlLiveRootCycles(count: number) { + drains.length = 0 + const timers = vi.spyOn(globalThis, 'setTimeout') + const settled = { answered: 0, failed: 0 } + const left: { timers: number; drainListeners: number }[] = [] + const waitsAfterExit: number[] = [] + for (let cycle = 0; cycle < count; cycle++) { + for (const ending of ['exit after EOF', 'no exit', 'close mid-drain'] as const) { + const opened = await TRANSPORTS.jsonl() + const { child } = opened + opened.pending.then( + () => (settled.answered += 1), + () => (settled.failed += 1) + ) + if (ending === 'close mid-drain') { + child.emit('exit', 1, null) + } else { + child.stdout.end() + await vi.advanceTimersByTimeAsync(0) + if (ending === 'exit after EOF') { + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS / 2) + child.emit('exit', 1, null) + await vi.advanceTimersByTimeAsync(0) + // The exit settles the call and ends the wait its output's end started. + waitsAfterExit.push(vi.getTimerCount()) + } else { + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS) + } + } + await vi.advanceTimersByTimeAsync(0) + const closing = opened.close() + if (ending === 'no exit') { + child.emit('exit', 0, null) + } + await closing + await vi.advanceTimersByTimeAsync(0) + left.push({ timers: vi.getTimerCount(), drainListeners: drainListeners() }) + child.emit('close', 0, null) + await vi.advanceTimersByTimeAsync(PROVIDER_EXIT_OUTPUT_DRAIN_MS) + } + } + const timersScheduled = timers.mock.calls.length + timers.mockRestore() + const finished = drains.map((drain) => drain.finished) + return { settled, left, waitsAfterExit, timersScheduled, drains: drains.length, finished } +} + +it('jsonl: output ending before a live root exits leaves no timer or listener, at N and 10N', async () => { + vi.useFakeTimers() + const n = await jsonlLiveRootCycles(10) + const tenN = await jsonlLiveRootCycles(100) + for (const run of [n, tenN]) { + for (const held of run.left) { + expect(held).toEqual({ timers: 0, drainListeners: 0 }) + } + expect(run.waitsAfterExit.every((waits) => waits === 0)).toBe(true) + expect(run.finished.every((finished) => finished === 1)).toBe(true) + } + // Every unanswered call fails exactly once, however its output and process ended. + expect(n.settled).toEqual({ answered: 0, failed: 30 }) + expect(tenN.settled).toEqual({ answered: 0, failed: 300 }) + expect(tenN.timersScheduled).toBe(10 * n.timersScheduled) + expect(tenN.drains).toBe(10 * n.drains) +}) diff --git a/src/main/provider-process/provider-exit-output-drain.ts b/src/main/provider-process/provider-exit-output-drain.ts new file mode 100644 index 000000000000..995bb0bf61e5 --- /dev/null +++ b/src/main/provider-process/provider-exit-output-drain.ts @@ -0,0 +1,39 @@ +import type { EventEmitter } from 'node:events' +import type { Readable } from 'node:stream' + +/** How long an exited child's output may still deliver answers before its pending calls fail. */ +export const PROVIDER_EXIT_OUTPUT_DRAIN_MS = 1_000 + +/** Process exit can be observed before stdout is read to its end, so an answer written just + * before exit would otherwise lose to the exit. Runs `finish` once stdout ends or closes, the + * child reports all its stdio closed, or the bound passes while a descendant still holds the + * pipe. Returns a way to finish now. */ +export function afterProviderOutputDrains( + stdout: Readable, + finish: () => void, + child?: EventEmitter +): () => void { + let finished = false + let timer: ReturnType | undefined + const done = (): void => { + if (finished) { + return + } + finished = true + clearTimeout(timer) + stdout.removeListener('end', done) + stdout.removeListener('close', done) + child?.removeListener('close', done) + finish() + } + if (stdout.destroyed || stdout.readableEnded) { + done() + return done + } + stdout.once('end', done) + stdout.once('close', done) + child?.once('close', done) + timer = setTimeout(done, PROVIDER_EXIT_OUTPUT_DRAIN_MS) + timer.unref?.() + return done +} diff --git a/src/main/provider-process/provider-output-end.ts b/src/main/provider-process/provider-output-end.ts new file mode 100644 index 000000000000..2edd23b40ad7 --- /dev/null +++ b/src/main/provider-process/provider-output-end.ts @@ -0,0 +1,59 @@ +import type { Readable, Writable } from 'node:stream' +import { afterProviderOutputDrains } from './provider-exit-output-drain' +import { detachProviderStreamErrorHandler } from './provider-stdio-error-boundary' + +/** The agent stopped reading what Orca writes. Nothing more is written, but calls it may already + * have answered settle only once its output ends or the drain bound passes. */ +export class ProviderOutputEnd { + error?: Error + private detached = false + + constructor( + private readonly output: Writable, + private readonly ended: (error: Error) => void, + private readonly closedError: () => Error + ) { + output.on('close', this.onClosed) + output.on('finish', this.onClosed) + output.on('error', this.onError) + } + + /** False when the output already ended; the first end is the one its calls settle with. */ + begin(error: Error): boolean { + if (this.error) { + return false + } + this.error = error + this.detach() + return true + } + + /** `close` runs now when no call waits, else once `wait` (by default the output drain) ends. */ + settle( + waiting: boolean, + input: Readable, + close: () => void, + wait?: (finish: () => void) => void + ): void { + if (!waiting) { + close() + } else if (wait) { + wait(close) + } else { + afterProviderOutputDrains(input, close) + } + } + + detach(): void { + if (this.detached) { + return + } + this.detached = true + this.output.removeListener('close', this.onClosed) + this.output.removeListener('finish', this.onClosed) + detachProviderStreamErrorHandler(this.output, this.onError) + } + + private readonly onClosed = (): void => this.ended(this.closedError()) + private readonly onError = (error: Error): void => this.ended(error) +} diff --git a/src/main/runtime/agent-session/structured-agent-session-adapter-start-failure.test.ts b/src/main/runtime/agent-session/structured-agent-session-adapter-start-failure.test.ts index 8ad5905e9908..45f547fe3d78 100644 --- a/src/main/runtime/agent-session/structured-agent-session-adapter-start-failure.test.ts +++ b/src/main/runtime/agent-session/structured-agent-session-adapter-start-failure.test.ts @@ -1,3 +1,4 @@ +import { claudeScriptedChild } from '../../claude/claude-scripted-child.test-fixture' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -32,7 +33,11 @@ afterEach(async () => { } }) -async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgentChild) { +async function failedStart( + agent: 'pi' | 'codex' | 'grok' | 'claude', + script: ScriptedAgentChild, + unanswered = false +) { directory = await mkdtemp(join(tmpdir(), 'orca-adapter-start-failure-')) const root = directory const host = await ensureStructuredAgentSessionHost({ @@ -55,9 +60,11 @@ async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgent agent, providerHandle: undefined, accountHome: - agent === 'pi' - ? { variable: 'PI_CODING_AGENT_DIR', path: root } - : { variable: 'GROK_HOME', path: root }, + agent === 'claude' + ? { variable: 'CLAUDE_CONFIG_DIR', path: root } + : agent === 'pi' + ? { variable: 'PI_CODING_AGENT_DIR', path: root } + : { variable: 'GROK_HOME', path: root }, location: { executionHostId: 'local', wslDistro: null, @@ -88,10 +95,15 @@ async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgent if (!sent.ok) { throw new Error(JSON.stringify(sent.refusal)) } - for (let attempt = 1; attempt <= 5; attempt++) { + const expectedAttempts = unanswered || agent === 'grok' || agent === 'claude' ? 5 : 1 + for (let attempt = 1; attempt <= expectedAttempts; attempt++) { // Retries back off 100ยท2^n ms (800 ms before the fifth spawn); the 1 s default leaves no room under load. await vi.waitFor(() => expect(script.spawns()).toBe(attempt), { timeout: 5_000 }) - script.failHandshake('provider config is invalid: bad key') + const fail = unanswered ? script.exitHandshake : script.failHandshake + if (!fail) { + throw new Error('fixture has no unanswered exit') + } + fail('provider config is invalid: bad key') } const submission = async () => { await host.flushStreamedEvents(SESSION) @@ -104,7 +116,7 @@ async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgent }) const settled = await submission() const snapshot = await host.journalSnapshot(SESSION) - const failureKind = agent === 'codex' ? 'startFailed' : 'providerStartFailed' + const failureKind = agent === 'codex' && !unanswered ? 'startFailed' : 'providerStartFailed' expect(readWholeAgentSessionFailureFact(settled?.rejection)).toMatchObject({ kind: failureKind, detail: { text: 'provider config is invalid: bad key' } @@ -117,7 +129,7 @@ async function failedStart(agent: 'pi' | 'codex' | 'grok', script: ScriptedAgent ) ).toHaveLength(1) expect(script.prompts()).toEqual([]) - expect(script.spawns()).toBe(5) + expect(script.spawns()).toBe(expectedAttempts) } it('Pi records one failure without dispatch after a failed start', async () => { @@ -129,3 +141,13 @@ it('Codex records one failure without dispatch after a failed start', async () = it('ACP records one failure without dispatch after a failed start', async () => { await failedStart('grok', acpScriptedChild('grok')()) }) + +it('Claude keeps unanswered early exits within five attempts', async () => { + await failedStart('claude', claudeScriptedChild()) +}) +it('Codex keeps unanswered early exits within five attempts', async () => { + await failedStart('codex', codexScriptedChild(), true) +}) +it('Pi keeps unanswered early exits within five attempts', async () => { + await failedStart('pi', piScriptedChild(), true) +}) diff --git a/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts b/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts index 9b4aa00de645..150f60ee1b8f 100644 --- a/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts +++ b/src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts @@ -11,6 +11,8 @@ import { setStructuredAgentSessionHost } from '../../../native-chat/agent-sessio import type { RpcContext } from '../core' import { AgentSessionPreSpawnError } from '../../../native-chat/agent-session-wire/structured-agent-session-adapter' import { CodexAppServerRequestError } from '../../../codex/app-server/codex-app-server-request-error' +import { CodexAppServerExitError } from '../../../codex/app-server/codex-app-server-exit-error' +import { codexStartFailure } from '../../../codex/app-server/codex-startup-outcome' import { executeAgentLaunch } from '../../../agent-launch/agent-launch-executor' import { agentLaunchSurfaceFactory } from './agent-launch-surfaces' import * as structuredCreate from './structured-agent-session-create' @@ -39,7 +41,9 @@ const CLI: Partial = {} const SESSION = 'codex_9b1deb4d_3b7d_4bad_9bdd_2b0d7b3dcb6d' let rig: RestTestRig +const openEnded = new Map void>() beforeEach(async () => { + openEnded.clear() // The launch mints its operation id from the wall clock; the rig's host reads a fixed one. vi.useFakeTimers({ now: HOST_TEST_NOW, toFake: ['Date'] }) rig = await createRestTestRig({ idleSweep: { intervalMs: 60_000 } }) @@ -135,12 +139,18 @@ async function child(sessionId = SESSION) { async function endChild(sessionId = SESSION) { const current = await child(sessionId) + await vi.waitFor(() => expect(openEnded.has(current.acquisitionGeneration)).toBe(true)) await rig.host.handleAdapterEvent({ type: 'ended', ...current, reason: 'exited during initialize', cause: 'unexpected-exit' }) + // The real adapter answers its outstanding open with the drained exit once the child exits. + openEnded.get(current.acquisitionGeneration)?.( + codexStartFailure(new CodexAppServerExitError('codex app-server connection ended')) + ) + openEnded.delete(current.acquisitionGeneration) await vi.waitFor(() => expect(rig.host.collaboratorsForTests().sessions.get(sessionId)?.child).toBeNull() ) @@ -149,6 +159,17 @@ async function endChild(sessionId = SESSION) { function holdStarts() { const acquire = rig.adapter.acquire.getMockImplementation()! rig.adapter.acquire.mockImplementation(rig.heldStart(acquire)) + const open = rig.host.deps.adapter.open + vi.spyOn(rig.host.deps.adapter, 'open').mockImplementation((input) => + Promise.race([ + open(input), + new Promise((_resolve, reject) => { + if (input.acquisitionGeneration) { + openEnded.set(input.acquisitionGeneration, reject) + } + }) + ]) + ) } it('a whitespace-only submit creates an eager empty chat without sending a turn', async () => { @@ -232,7 +253,14 @@ it.each(['spawn', 'open'] as const)( ) } else { vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValueOnce( - new CodexAppServerRequestError('initialize', -32603, 'temporarily unavailable') + codexStartFailure( + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) + ) ) } const result = await launch(CLI, 'fix the build') diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts index 81738e589010..a2fd787011e5 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-retry.test.ts @@ -7,6 +7,7 @@ import { import { HOST_TEST_NOW } from '../../../native-chat/agent-session-wire/structured-agent-session-host-test-data' import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import { CodexAppServerRequestError } from '../../../codex/app-server/codex-app-server-request-error' +import { codexStartFailure } from '../../../codex/app-server/codex-startup-outcome' import { OrchestrationDb } from '../../orchestration/db' import { createStructuredWorkerSession, @@ -48,6 +49,17 @@ afterEach(async () => { vi.useRealTimers() }) +// The Codex adapter's open answers with its typed verdict, which the host reads as given. +const overloaded = () => + codexStartFailure( + new CodexAppServerRequestError( + 'initialize', + -32001, + 'Server overloaded; retry later.', + 'Server overloaded; retry later.' + ) + ) + async function createWorker() { const workerRuntime = { ...runtimeStub(), @@ -112,9 +124,7 @@ it('starts only for its agent-sourced preamble and observes it after a transient const sessionId = created.identity.sessionId expect(rig.adapter.acquire).not.toHaveBeenCalled() expect((await rig.host.journalSnapshot(sessionId)).submissions).toEqual([]) - vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValueOnce( - new CodexAppServerRequestError('initialize', -32603, 'temporarily unavailable') - ) + vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValueOnce(overloaded()) const result = await deliverPreamble() expect(result.structuredTurnStart).toEqual({ verdict: 'observed' }) expect(rig.adapter.acquire).toHaveBeenCalledTimes(2) @@ -144,9 +154,7 @@ it('reports a preamble as undelivered after startup exhaustion and retires the f const { runtime, created, workerRuntime, deliverPreamble } = await createWorker() const sessionId = created.identity.sessionId expect(rig.store.listVisibleSessionIds()).toContain(sessionId) - vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValue( - new CodexAppServerRequestError('initialize', -32603, 'temporarily unavailable') - ) + vi.spyOn(rig.host.deps.adapter, 'open').mockRejectedValue(overloaded()) await expect(deliverPreamble()).rejects.toMatchObject({ code: 'dispatch_preamble_undelivered' }) expect(rig.adapter.acquire).toHaveBeenCalledTimes(5) expect(rig.adapter.dispatch).not.toHaveBeenCalled() diff --git a/src/main/runtime/structured-agent-scripted-child.test-fixture.ts b/src/main/runtime/structured-agent-scripted-child.test-fixture.ts index 1e1d8fd3a409..79b551addb11 100644 --- a/src/main/runtime/structured-agent-scripted-child.test-fixture.ts +++ b/src/main/runtime/structured-agent-scripted-child.test-fixture.ts @@ -11,6 +11,8 @@ export type ScriptedAgentChild = { releaseHandshake(): void /** Fails the held handshake of the newest spawn with this provider error; its process exits. */ failHandshake(message: string): void + /** The child dies without answering the pending startup request. */ + exitHandshake?: (message: string) => void /** Default prompts complete immediately; false keeps turns running until interrupt or child exit, modeling background work. */ completeTurns: boolean /** The text of every prompt any spawn received, in order. */ diff --git a/src/shared/main-process-ndjson-framer.ts b/src/shared/main-process-ndjson-framer.ts index 4c579ffebec4..4020486f2dba 100644 --- a/src/shared/main-process-ndjson-framer.ts +++ b/src/shared/main-process-ndjson-framer.ts @@ -37,6 +37,8 @@ export type IncrementalNdjsonFramer = { export type IncrementalNdjsonFramerOptions = { maxLineBytes?: number shouldPause?: () => boolean + /** Skips parsing lines the consumer can rule out from their text. */ + acceptLine?: (line: string) => boolean } function boundedUtf8Prefix(value: string, maxBytes: number): string { @@ -192,7 +194,7 @@ export function createIncrementalNdjsonFramer( lineSegments.push(segment) const line = lineSegments.length === 1 ? lineSegments[0] : lineSegments.join('') clearLine() - if (!/^\s*$/.test(line)) { + if (!/^\s*$/.test(line) && (options.acceptLine?.(line) ?? true)) { let parsed: unknown try { parsed = JSON.parse(line)