Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions src/main/ipc/runtime-environment-recovery-handler.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import { describe, expect, it, vi } from 'vitest'

const handlers = vi.hoisted(() => new Map<string, () => unknown>())
const retrySharedControl = vi.hoisted(() => vi.fn())
const retrySsh = vi.hoisted(() => vi.fn())

vi.mock('electron', () => ({
ipcMain: {
removeHandler: (channel: string) => handlers.delete(channel),
handle: (channel: string, handler: () => unknown) => handlers.set(channel, handler)
}
}))
vi.mock('./runtime-environment-request-connections', () => ({
retryRemoteRuntimeSharedControlConnectionsNow: retrySharedControl
}))
vi.mock('./ssh-host-sleep-reconnect', () => ({
retrySshConnectionsAfterNetworkReturn: retrySsh
}))

import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler'

describe('runtimeEnvironments:retryConnectionsNow', () => {
it('retries SSH transports as well as shared control when the window reports the network back', () => {
registerRuntimeEnvironmentRecoveryHandler()
handlers.get('runtimeEnvironments:retryConnectionsNow')?.()
expect(retrySharedControl).toHaveBeenCalledTimes(1)
expect(retrySsh).toHaveBeenCalledTimes(1)
})
})
3 changes: 3 additions & 0 deletions src/main/ipc/runtime-environment-recovery-handler.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import { ipcMain } from 'electron'
import { retryRemoteRuntimeSharedControlConnectionsNow } from './runtime-environment-request-connections'
import { retrySshConnectionsAfterNetworkReturn } from './ssh-host-sleep-reconnect'

const RETRY_CONNECTIONS_NOW_CHANNEL = 'runtimeEnvironments:retryConnectionsNow'

export function registerRuntimeEnvironmentRecoveryHandler(): void {
ipcMain.removeHandler(RETRY_CONNECTIONS_NOW_CHANNEL)
ipcMain.handle(RETRY_CONNECTIONS_NOW_CHANNEL, () => {
retryRemoteRuntimeSharedControlConnectionsNow()
// Why: the window's `online` event is main's only network-return signal; SSH needs it too.
retrySshConnectionsAfterNetworkReturn()
})
}
29 changes: 29 additions & 0 deletions src/main/ipc/ssh-host-sleep-reconnect.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it, vi } from 'vitest'

const recoverManagedTunnels = vi.hoisted(() => vi.fn(async () => {}))
const retryStalledReconnects = vi.hoisted(() => vi.fn((): string[] => []))

vi.mock('electron', async () => {
const { EventEmitter } = await import('node:events')
Expand All @@ -9,10 +10,14 @@ vi.mock('electron', async () => {
vi.mock('../ssh/orcad-managed-tunnel', () => ({
recoverOrcadManagedTunnelsAfterHostResume: recoverManagedTunnels
}))
vi.mock('../ssh/ssh-target-registry', () => ({
getSshConnectionManager: () => ({ retryStalledReconnects })
}))

import { powerMonitor } from 'electron'
import {
registerPowerMonitorReconnect,
retrySshConnectionsAfterNetworkReturn,
unregisterPowerMonitorReconnect
} from './ssh-host-sleep-reconnect'

Expand All @@ -39,4 +44,28 @@ describe('host sleep reconnect', () => {
await new Promise((resolve) => setImmediate(resolve))
expect(recoverManagedTunnels).not.toHaveBeenCalled()
})

it.each(['resume', 'unlock-screen'])(
'restarts stalled SSH reconnects on %s (#26888)',
(event) => {
registerPowerMonitorReconnect(() => '/user-data')
powerMonitor.emit(event)
expect(retryStalledReconnects).toHaveBeenCalledTimes(1)
}
)

it('restarts stalled SSH reconnects and re-probes tunnels when the window reports the network back', async () => {
registerPowerMonitorReconnect(() => '/user-data')
retrySshConnectionsAfterNetworkReturn()
expect(retryStalledReconnects).toHaveBeenCalledTimes(1)
await vi.waitFor(() => expect(recoverManagedTunnels).toHaveBeenCalledTimes(1))
})

it('stops listening once unregistered', () => {
registerPowerMonitorReconnect(() => '/user-data')
unregisterPowerMonitorReconnect()
powerMonitor.emit('resume')
powerMonitor.emit('unlock-screen')
expect(retryStalledReconnects).not.toHaveBeenCalled()
})
})
44 changes: 32 additions & 12 deletions src/main/ipc/ssh-host-sleep-reconnect.ts
Original file line number Diff line number Diff line change
@@ -1,34 +1,54 @@
import { powerMonitor } from 'electron'
import { recoverOrcadManagedTunnelsAfterHostResume } from '../ssh/orcad-managed-tunnel'
import { getSshConnectionManager } from '../ssh/ssh-target-registry'
import { errorMessage } from '../../shared/error-message'

let powerMonitorUnsubscribe: (() => void) | null = null
let registeredGetUserDataPath: (() => string) | undefined

// Why: macOS can resume before the network is back, so a failed first probe gets one retry before the link is declared dead (#7773).
const RESUME_PROBE_TIMEOUT_MS = 5_000
const RESUME_PROBE_ATTEMPTS = 2

/**
* Wake, unlock, or the window's resume/online signal. Why each: the ladder can spend itself in
* macOS dark wakes, which emit no resume, so only the next real signal can give it a fresh budget.
*/
export function retrySshConnectionsAfterNetworkReturn(): void {
const restarted = getSshConnectionManager()?.retryStalledReconnects() ?? []
if (restarted.length > 0) {
console.warn(`[ssh] Retrying ${restarted.length} stalled SSH reconnect(s) after wake/network`)
}
const getUserDataPath = registeredGetUserDataPath
if (!getUserDataPath) {
return
}
// Why: a transport that still looks connected after wake is found dead by this probe, not by keepalive.
void recoverOrcadManagedTunnelsAfterHostResume(getUserDataPath(), {
attempts: RESUME_PROBE_ATTEMPTS,
timeoutMs: RESUME_PROBE_TIMEOUT_MS
}).catch((err) => {
console.warn(
`[ssh] Failed to recover a managed Orca tunnel after system resume: ${errorMessage(err)}`
)
})
}

export function registerPowerMonitorReconnect(getUserDataPath?: () => string): void {
powerMonitorUnsubscribe?.()
const onResume = (): void => {
if (getUserDataPath) {
void recoverOrcadManagedTunnelsAfterHostResume(getUserDataPath(), {
attempts: RESUME_PROBE_ATTEMPTS,
timeoutMs: RESUME_PROBE_TIMEOUT_MS
}).catch((err) => {
console.warn(
`[ssh] Failed to recover a managed Orca tunnel after system resume: ${errorMessage(err)}`
)
})
}
}
registeredGetUserDataPath = getUserDataPath
const onResume = (): void => retrySshConnectionsAfterNetworkReturn()
powerMonitor.on('resume', onResume)
// Why: macOS and Windows only; a lock without sleep can still drop a VPN route such as Tailscale's.
powerMonitor.on('unlock-screen', onResume)
powerMonitorUnsubscribe = () => {
powerMonitor.off('resume', onResume)
powerMonitor.off('unlock-screen', onResume)
}
}

export function unregisterPowerMonitorReconnect(): void {
powerMonitorUnsubscribe?.()
powerMonitorUnsubscribe = null
registeredGetUserDataPath = undefined
}
11 changes: 11 additions & 0 deletions src/main/ssh/ssh-connection-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,17 @@ export class SshConnectionManager {
await conn.reconnect()
}

/** Restarts every reconnect stalled on a backoff, a spent ladder, or a pre-wake handshake. */
retryStalledReconnects(): string[] {
const restarted: string[] = []
for (const [targetId, conn] of this.connections) {
if (conn.retryAfterNetworkReturn()) {
restarted.push(targetId)
}
}
return restarted
}

getConnection(targetId: string): SshConnection | undefined {
return this.connections.get(targetId)
}
Expand Down
203 changes: 203 additions & 0 deletions src/main/ssh/ssh-connection-wake-retry.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
advanceToNextSshClient,
clientInstances,
connectWithFakeTimers,
emitSshEvent,
nextSshClientCreation,
resetSshConnectionMocks,
spawnSystemSshCommandMock,
ssh2Mock
} from './ssh-connection-test-harness'
import {
createCallbacks,
createHangingSystemCommandChannel,
createResolvedConfig,
createTarget
} from './ssh-connection-test-fixtures'
import { SshConnection } from './ssh-connection'
import { resolveWithSshG } from './ssh-config-parser'
import { SshConnectionManager } from './ssh-connection-manager'
import { RECONNECT_BACKOFF_MS } from './ssh-connection-utils'

vi.mock('ssh2', async () => (await import('./ssh-connection-test-harness')).createSsh2Module())
vi.mock('./system-ssh-binary', async () =>
(await import('./ssh-connection-test-harness')).createSystemSshBinaryModule()
)
vi.mock('./ssh-system-fallback', async () =>
(await import('./ssh-connection-test-harness')).createSystemFallbackModule()
)
vi.mock('./ssh-control-socket', async () =>
(await import('./ssh-connection-test-harness')).createControlSocketModule()
)
vi.mock('./ssh-config-parser', async () =>
(await import('./ssh-connection-test-harness')).createSshConfigParserModule()
)

function failHandshakes(): void {
ssh2Mock.connectBehavior = 'error'
ssh2Mock.connectErrorMessage = 'connect ETIMEDOUT 10.0.0.5:22'
ssh2Mock.connectErrorCode = 'ETIMEDOUT'
}

function recordStatuses(): { statuses: string[]; conn: SshConnection } {
const statuses: string[] = []
const conn = new SshConnection(
createTarget(),
createCallbacks({ onStateChange: vi.fn((_id, state) => statuses.push(state.status)) })
)
return { statuses, conn }
}

async function spendLadderWhileAsleep(conn: SshConnection): Promise<void> {
failHandshakes()
emitSshEvent('close')
for (const delayMs of RECONNECT_BACKOFF_MS) {
await advanceToNextSshClient(delayMs)
}
expect(conn.getState().status).toBe('reconnection-failed')
}

describe('SshConnection retry after wake or network return', () => {
beforeEach(() => {
resetSshConnectionMocks()
vi.useFakeTimers()
})
afterEach(() => {
vi.useRealTimers()
})

it('retries a host the ladder gave up on while the machine slept (#26888)', async () => {
const { conn } = recordStatuses()
await connectWithFakeTimers(conn)
await spendLadderWhileAsleep(conn)
const clientsBefore = clientInstances.length

// Nothing retries a spent ladder on its own.
await vi.advanceTimersByTimeAsync(10 * 60_000)
expect(clientInstances).toHaveLength(clientsBefore)

ssh2Mock.connectBehavior = 'ready'
const created = nextSshClientCreation()
expect(conn.retryAfterNetworkReturn()).toBe(true)
await created
await vi.advanceTimersByTimeAsync(1)

expect(conn.getState().status).toBe('connected')
})

it('gives a wake retry the full failure budget, not what the dark wakes left', async () => {
const { conn } = recordStatuses()
await connectWithFakeTimers(conn)
await spendLadderWhileAsleep(conn)

// The network is still coming back: the wake attempt fails and the ladder must start afresh.
const created = nextSshClientCreation()
conn.retryAfterNetworkReturn()
await created
await vi.advanceTimersByTimeAsync(1)
expect(conn.getState().status).toBe('reconnecting')

ssh2Mock.connectBehavior = 'ready'
await advanceToNextSshClient(RECONNECT_BACKOFF_MS[0])
expect(conn.getState().status).toBe('connected')
})

it('skips the backoff a reconnect is waiting on', async () => {
const { conn } = recordStatuses()
await connectWithFakeTimers(conn)
failHandshakes()
emitSshEvent('close')
// Climb to the 30 s step.
for (const delayMs of RECONNECT_BACKOFF_MS.slice(0, 7)) {
await advanceToNextSshClient(delayMs)
}
expect(conn.getState().status).toBe('reconnecting')

ssh2Mock.connectBehavior = 'ready'
const created = nextSshClientCreation()
expect(conn.retryAfterNetworkReturn()).toBe(true)
await created
await vi.advanceTimersByTimeAsync(1)
expect(conn.getState().status).toBe('connected')
})

it('replaces a background handshake left hanging on the route from before sleep', async () => {
const { conn } = recordStatuses()
await connectWithFakeTimers(conn)
ssh2Mock.connectBehavior = 'pending'
emitSshEvent('close')
await advanceToNextSshClient(RECONNECT_BACKOFF_MS[0])
expect(conn.getState().status).toBe('connecting')

// A handshake the first wake signal just started is left alone.
expect(conn.retryAfterNetworkReturn()).toBe(false)

await vi.advanceTimersByTimeAsync(10_000)
ssh2Mock.connectBehavior = 'ready'
const created = nextSshClientCreation()
expect(conn.retryAfterNetworkReturn()).toBe(true)
await created
await vi.advanceTimersByTimeAsync(1)
expect(conn.getState().status).toBe('connected')
})

it("keeps the replacement's system-SSH transport when the superseded probe closes late", async () => {
vi.mocked(resolveWithSshG).mockResolvedValue(createResolvedConfig())
const conn = new SshConnection(createTarget(), createCallbacks())
await conn.connect()
expect(conn.usesSystemSshTransport()).toBe(true)

const stale = createHangingSystemCommandChannel()
spawnSystemSshCommandMock.mockImplementationOnce(() => stale)
void conn.reconnect()
await vi.advanceTimersByTimeAsync(10_000)
expect(conn.getState().status).toBe('connecting')

expect(conn.retryAfterNetworkReturn()).toBe(true)
await vi.advanceTimersByTimeAsync(1)
expect(conn.getState().status).toBe('connected')

// Closing a system-SSH channel only signals the process; its close lands after the replacement.
stale.emit('close', null)
await vi.advanceTimersByTimeAsync(1)
expect(conn.getState().status).toBe('connected')
expect(conn.usesSystemSshTransport()).toBe(true)
})

it('leaves healthy, initial, and explicitly closed connections alone', async () => {
const { conn } = recordStatuses()
await connectWithFakeTimers(conn)
expect(conn.retryAfterNetworkReturn()).toBe(false)

await conn.disconnect()
expect(conn.retryAfterNetworkReturn()).toBe(false)

const initial = new SshConnection(createTarget(), createCallbacks())
ssh2Mock.connectBehavior = 'pending'
void initial.connect().catch(() => undefined)
await vi.advanceTimersByTimeAsync(10_000)
// The first connect owns its own retries and result.
expect(initial.retryAfterNetworkReturn()).toBe(false)
await initial.disconnect()
})

it('restarts every stalled connection the manager holds', async () => {
const manager = new SshConnectionManager(createCallbacks())
const created = nextSshClientCreation()
const connecting = manager.connect(createTarget())
await created
await vi.advanceTimersByTimeAsync(1)
const conn = await connecting
await spendLadderWhileAsleep(conn)

ssh2Mock.connectBehavior = 'ready'
const recreated = nextSshClientCreation()
expect(manager.retryStalledReconnects()).toEqual([createTarget().id])
await recreated
await vi.advanceTimersByTimeAsync(1)
expect(manager.getState(createTarget().id)?.status).toBe('connected')
expect(manager.retryStalledReconnects()).toEqual([])
await manager.disconnectAll()
})
})
Loading
Loading