Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1453,6 +1453,181 @@ describe('createRemoteRuntimePtyTransport', () => {
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2))
})

it('retries a transient wake failure without surfacing a PTY error', async () => {
let subscribeAttempt = 0
const transportCallbacks: NonNullable<typeof subscriptionCallbacks>[] = []
runtimeSubscribe.mockImplementation(
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
subscribeAttempt += 1
transportCallbacks.push(callbacks)
subscriptionCallbacks = callbacks
if (subscribeAttempt === 2) {
throw new Error('Could not connect to the remote Orca runtime.')
}
queueMicrotask(emitMultiplexReady)
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
}
)
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})

await transport.connect({ url: '', callbacks: { onError } })
transportCallbacks[0].onError?.({
code: 'remote_runtime_unavailable',
message: 'Remote Orca runtime stopped responding; the stream connection was reset.'
})

await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(3))
expect(onError).not.toHaveBeenCalled()
transport.destroy?.()
})

it('retries a restored pane when the runtime is still unavailable', async () => {
let subscribeAttempt = 0
runtimeSubscribe.mockImplementation(
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
subscribeAttempt += 1
subscriptionCallbacks = callbacks
if (subscribeAttempt === 1) {
throw new Error('Could not connect to the remote Orca runtime.')
}
queueMicrotask(emitMultiplexReady)
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
}
)
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})

transport.attach({
existingPtyId: 'remote:terminal-1',
callbacks: { onError }
})

await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2))
expect(onError).not.toHaveBeenCalled()
transport.destroy?.()
})

it('retries when recoverable transport error arrives before multiplex ready', async () => {
let subscribeAttempt = 0
runtimeSubscribe.mockImplementation(
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
subscribeAttempt += 1
subscriptionCallbacks = callbacks
if (subscribeAttempt === 1) {
queueMicrotask(() =>
callbacks.onError?.({
code: 'remote_runtime_unavailable',
message: 'Remote Orca runtime stopped responding; the stream connection was reset.'
})
)
} else {
queueMicrotask(emitMultiplexReady)
}
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
}
)
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})

transport.attach({ existingPtyId: 'remote:terminal-1', callbacks: { onError } })

await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2))
expect(onError).not.toHaveBeenCalled()
transport.destroy?.()
})

it('does not let a stale reconnect rearm retries after reattach', async () => {
let rejectStaleReconnect = (_error: Error): void => {}
const transportCallbacks: NonNullable<typeof subscriptionCallbacks>[] = []
runtimeSubscribe.mockImplementation(
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
transportCallbacks.push(callbacks)
subscriptionCallbacks = callbacks
if (transportCallbacks.length === 2) {
return await new Promise((_, reject) => {
rejectStaleReconnect = reject
})
}
queueMicrotask(emitMultiplexReady)
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
}
)
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})

await transport.connect({ url: '', callbacks: { onError } })
transportCallbacks[0].onClose?.()
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2))
transport.detach?.()
transport.attach({ existingPtyId: 'remote:terminal-1', callbacks: { onError } })
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(3))

rejectStaleReconnect(new Error('Could not connect to the remote Orca runtime.'))
await new Promise((resolve) => setTimeout(resolve, 350))

expect(runtimeSubscribe).toHaveBeenCalledTimes(3)
expect(onError).not.toHaveBeenCalled()
transport.destroy?.()
})

it('does not let a stale same-handle attach rearm retries after reattach', async () => {
let rejectStaleAttach = (_error: Error): void => {}
runtimeSubscribe.mockImplementation(
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
subscriptionCallbacks = callbacks
if (runtimeSubscribe.mock.calls.length === 1) {
return await new Promise((_, reject) => {
rejectStaleAttach = reject
})
}
queueMicrotask(emitMultiplexReady)
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
}
)
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})

transport.attach({ existingPtyId: 'remote:terminal-1', callbacks: { onError } })
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(1))
transport.detach?.()
transport.attach({ existingPtyId: 'remote:terminal-1', callbacks: { onError } })
expect(runtimeSubscribe).toHaveBeenCalledTimes(1)

rejectStaleAttach(new Error('Could not connect to the remote Orca runtime.'))
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2))
await new Promise((resolve) => setTimeout(resolve, 350))

expect(runtimeSubscribe).toHaveBeenCalledTimes(2)
expect(onError).not.toHaveBeenCalled()
transport.destroy?.()
})

it('releases pending claimed input when reconnect subscription fails', async () => {
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { unwrapRuntimeRpcResult } from '../../runtime/runtime-rpc-client'
import {
getRemoteRuntimePtyEnvironmentId,
getRemoteRuntimeTerminalHandle,
isRecoverableRuntimeTransportError,
runtimeTerminalErrorMessage,
toRemoteRuntimePtyId
} from '../../runtime/runtime-terminal-stream'
Expand All @@ -41,6 +42,7 @@ const REMOTE_TERMINAL_INPUT_FLUSH_MS = 8
const REMOTE_TERMINAL_VIEWPORT_FLUSH_MS = 33
const HOST_SESSION_ATTACH_POLL_MS = 150
const HOST_SESSION_ATTACH_TIMEOUT_MS = 15_000
const REMOTE_TERMINAL_RESUBSCRIBE_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000]

function isRemoteTerminalGoneMessage(message: string): boolean {
return (
Expand Down Expand Up @@ -91,6 +93,9 @@ export function createRemoteRuntimePtyTransport(
let storedCallbacks: Parameters<PtyTransport['connect']>[0]['callbacks'] = {}
let resubscribing = false
let resubscribeRequested = false
let resubscribeRetryAttempt = 0
let resubscribeRetryTimer: ReturnType<typeof setTimeout> | null = null
let resubscribeCycle = 0
let subscriptionGeneration = 0
let pendingViewportClaim = false
let pendingClaimInput = ''
Expand Down Expand Up @@ -404,6 +409,7 @@ export function createRemoteRuntimePtyTransport(

function retireRemoteTerminalId(): void {
connected = false
stopResubscribeRetries()
clearPendingViewportClaim()
const stalePtyId = remotePtyId
handle = null
Expand All @@ -416,6 +422,11 @@ export function createRemoteRuntimePtyTransport(

function handleRemoteTerminalError(error: unknown): void {
const message = runtimeTerminalErrorMessage(error)
if (isRecoverableRuntimeTransportError(error)) {
// Why: the liveness path owns recovery; transient sleep/network gaps are
// not terminal failures and must not leave a red banner in the PTY.
return
}
if (message === REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE) {
// Why: an oversized initial snapshot is skipped but live output keeps
// flowing — informational, not fatal, so never surface a red xterm banner.
Expand All @@ -435,10 +446,16 @@ export function createRemoteRuntimePtyTransport(
// handle (reconnect, epoch or PTY change). Re-derive it from the current
// session snapshot instead of resubscribing the stale closure value, which
// would mirror (and type into) whatever PTY now sits behind it (#7718).
async function resubscribeAfterTransportClose(previousHandle: string): Promise<void> {
async function resubscribeAfterTransportClose(
previousHandle: string,
cycle: number
): Promise<void> {
if (cycle !== resubscribeCycle) {
return
}
if (tabId && isWebTerminalSurfaceTabId(tabId)) {
const nextHandle = await listHostSessionHandle(toHostSessionTabId(tabId))
if (destroyed || !connected || handle !== previousHandle) {
if (cycle !== resubscribeCycle || destroyed || !connected || handle !== previousHandle) {
return
}
if (!nextHandle) {
Expand All @@ -453,9 +470,39 @@ export function createRemoteRuntimePtyTransport(
onPtySpawn?.(remotePtyId)
}
}
if (cycle !== resubscribeCycle) {
return
}
await subscribeToHandle()
}

function stopResubscribeRetries(): void {
resubscribeCycle += 1
if (resubscribeRetryTimer) {
clearTimeout(resubscribeRetryTimer)
resubscribeRetryTimer = null
}
resubscribeRetryAttempt = 0
resubscribeRequested = false
resubscribing = false
}

function scheduleResubscribeRetry(): void {
if (resubscribeRetryTimer || destroyed || !connected || !handle) {
return
}
const delay =
REMOTE_TERMINAL_RESUBSCRIBE_DELAYS_MS[
Math.min(resubscribeRetryAttempt, REMOTE_TERMINAL_RESUBSCRIBE_DELAYS_MS.length - 1)
]
resubscribeRetryAttempt += 1
resubscribeRequested = false
resubscribeRetryTimer = setTimeout(() => {
resubscribeRetryTimer = null
scheduleResubscribeAfterTransportClose()
}, delay)
}

function scheduleResubscribeAfterTransportClose(): void {
if (destroyed || !connected || !handle) {
return
Expand All @@ -464,18 +511,36 @@ export function createRemoteRuntimePtyTransport(
resubscribeRequested = true
return
}
if (resubscribeRetryTimer) {
return
}
resubscribing = true
const cycle = resubscribeCycle
const resubscribeHandle = handle
void resubscribeAfterTransportClose(resubscribeHandle)
void resubscribeAfterTransportClose(resubscribeHandle, cycle)
.then(() => {
if (cycle === resubscribeCycle) {
resubscribeRetryAttempt = 0
}
})
.catch((error) => {
if (!destroyed && connected && handle) {
if (cycle === resubscribeCycle && !destroyed && connected && handle) {
clearPendingViewportClaim()
handleRemoteTerminalError(error)
if (isRecoverableRuntimeTransportError(error)) {
// Why: Tailscale/Wi-Fi can lag behind wake. Keep the pane alive and
// retry at a capped cadence until the runtime is reachable again.
scheduleResubscribeRetry()
} else {
handleRemoteTerminalError(error)
}
}
})
.finally(() => {
if (cycle !== resubscribeCycle) {
return
}
resubscribing = false
if (resubscribeRequested) {
if (resubscribeRequested && !resubscribeRetryTimer) {
resubscribeRequested = false
scheduleResubscribeAfterTransportClose()
}
Expand Down Expand Up @@ -537,6 +602,7 @@ export function createRemoteRuntimePtyTransport(
}
outputProcessor.clearAccumulatedState()
connected = false
stopResubscribeRetries()
handle = null
remotePtyId = null
multiplexedStream = null
Expand Down Expand Up @@ -688,6 +754,8 @@ export function createRemoteRuntimePtyTransport(
},

attach(options) {
stopResubscribeRetries()
const attachCycle = resubscribeCycle
storedCallbacks = options.callbacks
currentRuntimeEnvironmentId =
getRemoteRuntimePtyEnvironmentId(options.existingPtyId) ?? runtimeEnvironmentId
Expand Down Expand Up @@ -715,15 +783,25 @@ export function createRemoteRuntimePtyTransport(
}
const targetHandle = handle
const targetPtyId = remotePtyId
void subscribeToHandle().catch((error) => {
if (!isCurrentRemoteTerminal(targetHandle, targetPtyId)) {
const attachSubscription = subscribeToHandle()
const attachSubscriptionGeneration = subscriptionGeneration
void attachSubscription.catch((error) => {
if (
attachCycle !== resubscribeCycle ||
attachSubscriptionGeneration !== subscriptionGeneration ||
!isCurrentRemoteTerminal(targetHandle, targetPtyId)
) {
return
}
if (handle === targetHandle && multiplexedStreamHandle !== targetHandle) {
closeMultiplexedStream()
}
clearPendingViewportClaim()
handleRemoteTerminalError(error)
if (isRecoverableRuntimeTransportError(error)) {
scheduleResubscribeAfterTransportClose()
} else {
handleRemoteTerminalError(error)
}
})
},

Expand All @@ -732,6 +810,7 @@ export function createRemoteRuntimePtyTransport(
inputBatcher.clear()
viewportBatcher.flush()
outputProcessor.clearAccumulatedState()
stopResubscribeRetries()
if (!connected && !handle) {
return
}
Expand All @@ -752,6 +831,7 @@ export function createRemoteRuntimePtyTransport(
inputBatcher.clear()
viewportBatcher.flush()
outputProcessor.clearAccumulatedState()
stopResubscribeRetries()
connected = false
clearPendingViewportClaim()
closeMultiplexedStream()
Expand Down
Loading