Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 165 additions & 0 deletions src/main/ipc/pty/pane/stable-pane-relay-absence-respawn.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
import { describe, expect, it, vi } from 'vitest'
import type { WorkspaceSessionState } from '../../../../shared/workspace-session-state-types'
import {
SSH_SESSION_EXPIRED_ERROR,
SshPtyAbsentFromRelayError
} from '../../../providers/ssh-pty-errors'
import type { Store } from '../../../persistence'
import type { IPtyProvider } from '../../../providers/types'
import { spawnForStablePane, type StablePaneOwner } from './stable-owner'

const LEAF = '1b3f2c4d-5e6a-4b7c-8d9e-0f1a2b3c4d5e'
const SIBLING_LEAF = '2c4d3e5f-6a7b-4c8d-9e0f-1a2b3c4d5e6f'
const WORKTREE = 'worktree-1'
const OWNER: StablePaneOwner = {
tabId: 'tab-1',
leafId: LEAF,
ptyId: 'ssh:conn-1@@pty-1',
hasPersistedBinding: true
}

function spawnAfterAttachRejection(
error: unknown,
persistence?: { store: Store; worktreeId: string }
): {
run: () => ReturnType<typeof spawnForStablePane>
spawn: ReturnType<typeof vi.fn>
} {
const spawn = vi
.fn()
.mockRejectedValueOnce(error)
.mockResolvedValueOnce({ id: 'ssh:conn-1@@pty-2', isReattach: false })
return {
spawn,
run: () =>
spawnForStablePane({
runtime: undefined,
provider: { spawn } as unknown as IPtyProvider,
spawnOptions: { cols: 80, rows: 24 },
owner: OWNER,
connectionId: 'conn-1',
resolveOwner: () => null,
...(persistence ? { store: persistence.store, worktreeId: persistence.worktreeId } : {})
})
}
}

/** Real retirement against an in-memory session, so `retireTerminalSurfaceFromPersistence` runs. */
function sessionStore(leaves: string[]): { store: Store; read: () => WorkspaceSessionState } {
let session = {
tabsByWorktree: {
[WORKTREE]: [{ id: 'tab-1', worktreeId: WORKTREE, ptyId: 'ssh:conn-1@@pty-1' }]
},
terminalLayoutsByTabId: {
'tab-1': {
root: leaves.reduce<Record<string, unknown>>(
(node, leafId) =>
Object.keys(node).length === 0
? { type: 'leaf', leafId }
: { type: 'split', direction: 'row', first: node, second: { type: 'leaf', leafId } },
{}
),
activeLeafId: leaves[0],
ptyIdsByLeafId: Object.fromEntries(
leaves.map((leafId, index) => [leafId, `ssh:conn-1@@pty-${index + 1}`])
)
}
},
terminalPtyIncarnationsByPaneKey: {}
} as unknown as WorkspaceSessionState
return {
read: () => session,
store: {
getWorkspaceSession: () => session,
setWorkspaceSession: (next: WorkspaceSessionState) => {
session = next
},
flushOrThrow: () => {}
} as unknown as Store
}
}

describe('stable pane adoption after the relay reports the PTY absent', () => {
it('spawns fresh once the relay has positively answered for that id', async () => {
const { run, spawn } = spawnAfterAttachRejection(
new SshPtyAbsentFromRelayError(`${SSH_SESSION_EXPIRED_ERROR}: pty-1`)
)

const result = await run()

expect(spawn).toHaveBeenCalledTimes(2)
expect(spawn.mock.calls[0]?.[0]).toMatchObject({ attachOnly: true })
expect(spawn.mock.calls[1]?.[0]).not.toHaveProperty('sessionId')
expect(result.owner).toBeNull()
})

// A restarted relay renumbers from pty-1, so the message alone cannot distinguish absence from a
// lost link — only the class may authorise abandoning the binding.
it.each([
['an expired session with no relay evidence', `${SSH_SESSION_EXPIRED_ERROR}: pty-1`],
['a lost link', 'SSH connection lost, reconnecting...'],
['a request timeout', 'Request "pty.attach" timed out after 10000ms']
])('keeps the binding and refuses to respawn after %s', async (_label, message) => {
const { run, spawn } = spawnAfterAttachRejection(new Error(message))

await expect(run()).rejects.toThrow(message)
expect(spawn).toHaveBeenCalledTimes(1)
})

// Why a real store: without one the `args.worktreeId` guard short-circuits and the retirement —
// which can delete the parent tab and its layout — never runs. "Reconnect lost every tab" is the
// regression this subsystem was reverted for twice, so the composition needs its own coverage.
describe('with persistence actually reached', () => {
it('retires only the absent leaf and leaves the tab and its sibling bound', async () => {
const { store, read } = sessionStore([LEAF, SIBLING_LEAF])
const { run, spawn } = spawnAfterAttachRejection(
new SshPtyAbsentFromRelayError(`${SSH_SESSION_EXPIRED_ERROR}: pty-1`),
{ store, worktreeId: WORKTREE }
)

const result = await run()

expect(spawn).toHaveBeenCalledTimes(2)
expect(result.owner).toBeNull()
const session = read()
expect(session.tabsByWorktree[WORKTREE]?.map((tab) => tab.id)).toEqual(['tab-1'])
const layout = session.terminalLayoutsByTabId['tab-1']
expect(layout).toBeDefined()
expect(layout?.ptyIdsByLeafId?.[SIBLING_LEAF]).toBe('ssh:conn-1@@pty-2')
expect(layout?.ptyIdsByLeafId?.[LEAF]).toBeUndefined()
})

// Pins current behaviour rather than blessing it: retiring the LAST leaf drops the tab from
// persistence, and the fallback returns owner=null so main does not re-persist a binding — tab
// survival then rests entirely on the renderer. If that ever regresses, this is the tripwire.
it('drops the tab when the absent leaf was the only one, leaving re-persistence to the renderer', async () => {
const { store, read } = sessionStore([LEAF])
const { run, spawn } = spawnAfterAttachRejection(
new SshPtyAbsentFromRelayError(`${SSH_SESSION_EXPIRED_ERROR}: pty-1`),
{ store, worktreeId: WORKTREE }
)

const result = await run()

expect(spawn).toHaveBeenCalledTimes(2)
expect(result.owner).toBeNull()
const session = read()
expect(session.tabsByWorktree[WORKTREE]).toEqual([])
expect(session.terminalLayoutsByTabId['tab-1']).toBeUndefined()
})

it('leaves persistence untouched when the failure is not positive absence', async () => {
const { store, read } = sessionStore([LEAF, SIBLING_LEAF])
const before = JSON.stringify(read())
const { run, spawn } = spawnAfterAttachRejection(
new Error('SSH connection lost, reconnecting...'),
{ store, worktreeId: WORKTREE }
)

await expect(run()).rejects.toThrow('SSH connection lost')

expect(spawn).toHaveBeenCalledTimes(1)
expect(JSON.stringify(read())).toBe(before)
})
})
})
13 changes: 11 additions & 2 deletions src/main/ipc/pty/provider/liveness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ import type { AgentSessionOwnerBinding } from '../../../../shared/agent-session-
import { agentSessionOwnerBindingsEqual } from '../../../../shared/claimed-agent-pty-owner'
import { addNodePtyRecoveryHint } from '../../../daemon/node-pty-error-hints'
import type { Store } from '../../../persistence'
import { isSshPtyNotFoundError } from '../../../providers/ssh-pty-errors'
import {
isSshPtyAbsentFromRelayError,
isSshPtyNotFoundError
} from '../../../providers/ssh-pty-errors'
import type { IPtyProvider } from '../../../providers/types'
import { markClaudePtyExited } from '../../../claude-accounts/live-pty-gate'
import { ptyIncarnationById, ptyOwnership } from './ownership-state'
Expand Down Expand Up @@ -54,7 +57,13 @@ export function normalizeNodePtySpawnError(err: unknown): Error {

export function isPtyAlreadyGoneError(err: unknown): boolean {
const message = err instanceof Error ? err.message : String(err)
return isSshPtyNotFoundError(err) || /Session not found/i.test(message)
return (
isSshPtyNotFoundError(err) ||
// Why: the reattach path rewrites the relay's wording to SSH_SESSION_EXPIRED and only this
// class preserves that the relay itself answered "absent" rather than the link dropping.
isSshPtyAbsentFromRelayError(err) ||
/Session not found/i.test(message)
)
}

export function delay(ms: number): Promise<void> {
Expand Down
22 changes: 22 additions & 0 deletions src/main/providers/ssh-pty-errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,25 @@ export function isSshPtyIdentityMismatchError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error)
return message.includes(SSH_PTY_IDENTITY_MISMATCH_ERROR) || /identity mismatch/i.test(message)
}

/**
* A reachable relay answered for this exact PTY id and reported it absent — positive evidence of
* absence from the execution host, so `exited` rather than `unverifiable`
* (docs/reference/ssh-execution-boundary.md). Deliberately NOT raised for a transport failure, a
* request timeout, a disposed multiplexer, an identity mismatch (the id names a live PTY belonging
* to another pane), or `restoreRequired` (the PTY is live, only its source stream is not) — none of
* those observe the process, and treating them as absence orphans live remote work.
*
* Carries the same `SSH_SESSION_EXPIRED` message so message-based consumers are unaffected; only
* callers that can act on the stronger verdict test the class.
*/
export class SshPtyAbsentFromRelayError extends Error {
constructor(message: string) {
super(message)
this.name = 'SshPtyAbsentFromRelayError'
}
}

export function isSshPtyAbsentFromRelayError(error: unknown): boolean {
return error instanceof SshPtyAbsentFromRelayError
}
78 changes: 78 additions & 0 deletions src/main/providers/ssh-pty-relay-absence-verdict.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import { describe, expect, it, vi } from 'vitest'
import {
SSH_PTY_IDENTITY_MISMATCH_ERROR,
SSH_SESSION_EXPIRED_ERROR,
isSshPtyAbsentFromRelayError
} from './ssh-pty-errors'
import { SshPtyProvider } from './ssh-pty-provider'

function providerRejectingAttachWith(error: unknown): SshPtyProvider {
const mux = {
request: vi.fn().mockRejectedValue(error),
notify: vi.fn(),
onNotification: vi.fn().mockReturnValue(vi.fn())
}
return new SshPtyProvider('conn-1', mux as never)
}

async function reattachRejection(error: unknown): Promise<unknown> {
const provider = providerRejectingAttachWith(error)
return await provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-1' }).then(
() => undefined,
(rejection: unknown) => rejection
)
}

describe('SSH PTY relay absence verdict', () => {
it('reports a relay-delivered not-found as positive absence', async () => {
const rejection = await reattachRejection(new Error('PTY "pty-1" not found'))

expect(isSshPtyAbsentFromRelayError(rejection)).toBe(true)
expect((rejection as Error).message).toBe(`${SSH_SESSION_EXPIRED_ERROR}: pty-1`)
})

it('does not report absence when the id names a live PTY owned by another pane', async () => {
const rejection = await reattachRejection(
new Error('PTY "pty-1" not found (identity mismatch)')
)

expect(isSshPtyAbsentFromRelayError(rejection)).toBe(false)
// Byte-identical to the pre-change message: only the class is new, so the toast humanizer,
// both spawn-execute lease paths, and the renderer's substring checks are untouched.
expect((rejection as Error).message).toBe(
`${SSH_SESSION_EXPIRED_ERROR}: pty-1 ${SSH_PTY_IDENTITY_MISMATCH_ERROR}`
)
})

// Loss of contact never observes the process: each of these must stay `unverifiable`.
it.each([
['a lost link', 'SSH connection lost, reconnecting...'],
['a disposed multiplexer', 'Multiplexer disposed'],
['a request timeout', 'Request "pty.attach" timed out after 10000ms']
])('does not report absence for %s', async (_label, message) => {
const rejection = await reattachRejection(new Error(message))

expect(isSshPtyAbsentFromRelayError(rejection)).toBe(false)
expect((rejection as Error).message).toBe(message)
})

it('does not report absence when the PTY is live but its source needs restoring', async () => {
const mux = {
request: vi.fn().mockResolvedValue({
incarnationId: 'incarnation-1',
sourceRecovery: { status: 'restoreRequired', reason: 'checkpointUnavailable' }
}),
notify: vi.fn(),
onNotification: vi.fn().mockReturnValue(vi.fn())
}
const provider = new SshPtyProvider('conn-1', mux as never)

const rejection = await provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-1' }).then(
() => undefined,
(error: unknown) => error
)

expect(isSshPtyAbsentFromRelayError(rejection)).toBe(false)
expect((rejection as Error).message).toContain(SSH_SESSION_EXPIRED_ERROR)
})
})
15 changes: 11 additions & 4 deletions src/main/providers/ssh-pty-session-reattach.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { isPtyIncarnationId, type PtyIncarnationId } from '../../shared/pty-inca
import {
SSH_PTY_IDENTITY_MISMATCH_ERROR,
SSH_SESSION_EXPIRED_ERROR,
SshPtyAbsentFromRelayError,
isSshPtyIdentityMismatchError,
isSshPtyNotFoundError
} from './ssh-pty-errors'
Expand Down Expand Up @@ -225,10 +226,16 @@ export async function reattachSshPtySession(args: {
// Why: an expired relay lease must be surfaced distinctly so the renderer clears its binding.
console.warn(`[ssh-pty] pty.attach FAILED for ${args.sessionId}:`, error)
if (isSshPtyNotFoundError(error)) {
const mismatchMarker = isSshPtyIdentityMismatchError(error)
? ` ${SSH_PTY_IDENTITY_MISMATCH_ERROR}`
: ''
throw new Error(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}${mismatchMarker}`)
if (isSshPtyIdentityMismatchError(error)) {
// The id names a LIVE PTY owned by another pane, so this is not evidence of absence.
throw new Error(
`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId} ${SSH_PTY_IDENTITY_MISMATCH_ERROR}`
)
}
// Why the class: the relay answered for this exact id, so callers holding a pane binding may
// retire it and spawn fresh. Plain `SSH_SESSION_EXPIRED` cannot say that — a restarted relay
// renumbers from pty-1, so the message alone is indistinguishable from a lost link.
throw new SshPtyAbsentFromRelayError(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}`)
}
throw error
}
Expand Down
Loading