Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 186 additions & 0 deletions src/main/ssh/ssh-relay-session-managed-hooks.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,4 +202,190 @@ describe('SshRelaySession managed hooks', () => {
session.dispose()
expect(cleanup).toHaveBeenCalledOnce()
})

// Why: a timed-out install left the relay on the old sources (a disabled agent kept its plugin) until the next reconnect.
it('retries a settings-triggered plugin install that failed, using the latest settings', async () => {
muxRequestMock.mockResolvedValue({ agents: [] })
const { mockStore, mockConn, mockPortForward, getMainWindow } = createMockDeps()
const settings = getDefaultSettings('/synthetic-home')
mockStore.getSettings = () => settings
let listener: Parameters<Store['onSettingsChanged']>[0] | undefined
mockStore.onSettingsChanged = (callback) => {
listener = callback
return vi.fn()
}
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const session = new SshRelaySession(
'target-settings',
getMainWindow,
mockStore,
mockPortForward
)
await session.establish(mockConn)
const installCalls = () =>
muxRequestMock.mock.calls.filter(([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD)

vi.useFakeTimers()
try {
const before = installCalls().length
muxRequestMock.mockImplementationOnce(async (method: string) => {
if (method !== AGENT_HOOK_INSTALL_PLUGINS_METHOD) {
return { agents: [] }
}
throw Object.assign(new Error('timed out'), { code: 'SSH_MUX_REQUEST_TIMEOUT' })
})
settings.disabledTuiAgents = ['opencode2']
listener?.({ disabledTuiAgents: settings.disabledTuiAgents }, settings)
await vi.advanceTimersByTimeAsync(60_000)

const calls = installCalls().slice(before)
expect(calls.length).toBeGreaterThan(1)
expect(calls.at(-1)?.[1]).toMatchObject({ opencode2PluginSource: '' })
} finally {
vi.useRealTimers()
warn.mockRestore()
session.dispose()
}
})

// Why: two overlapping failed installs must leave one retry chain, not two the next install can't cancel.
it('retries once when overlapping settings-triggered installs both fail', async () => {
muxRequestMock.mockResolvedValue({ agents: [] })
const { mockStore, mockConn, mockPortForward, getMainWindow } = createMockDeps()
const settings = getDefaultSettings('/synthetic-home')
mockStore.getSettings = () => settings
let listener: Parameters<Store['onSettingsChanged']>[0] | undefined
mockStore.onSettingsChanged = (callback) => {
listener = callback
return vi.fn()
}
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const session = new SshRelaySession(
'target-settings',
getMainWindow,
mockStore,
mockPortForward
)
await session.establish(mockConn)
const installCalls = () =>
muxRequestMock.mock.calls.filter(([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD)

vi.useFakeTimers()
try {
const before = installCalls().length
const pendingInstalls: ((error: Error) => void)[] = []
muxRequestMock.mockImplementation((method: string) => {
if (method === AGENT_HOOK_INSTALL_PLUGINS_METHOD && pendingInstalls.length < 2) {
return new Promise((_resolve, reject) => pendingInstalls.push(reject))
}
return Promise.resolve({ agents: [] })
})
settings.disabledTuiAgents = ['opencode2']
listener?.({ disabledTuiAgents: settings.disabledTuiAgents }, settings)
settings.disabledTuiAgents = ['opencode']
listener?.({ disabledTuiAgents: settings.disabledTuiAgents }, settings)
expect(pendingInstalls).toHaveLength(2)

for (const reject of pendingInstalls) {
reject(Object.assign(new Error('timed out'), { code: 'SSH_MUX_REQUEST_TIMEOUT' }))
}
await vi.advanceTimersByTimeAsync(60_000)

expect(installCalls().slice(before)).toHaveLength(3)
} finally {
vi.useRealTimers()
warn.mockRestore()
session.dispose()
}
})
it.each(['PERMISSION_DENIED', -32601, 'CONNECTION_LOST', 'DISPOSED'])(
'does not retry permanent or retired transport failure %s',
async (code) => {
muxRequestMock.mockResolvedValue({ agents: [] })
const { mockStore, mockConn, mockPortForward, getMainWindow } = createMockDeps()
const settings = getDefaultSettings('/synthetic-home')
mockStore.getSettings = () => settings
let listener: Parameters<Store['onSettingsChanged']>[0] | undefined
mockStore.onSettingsChanged = (callback) => {
listener = callback
return vi.fn()
}
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const session = new SshRelaySession(
'target-no-retry',
getMainWindow,
mockStore,
mockPortForward
)
await session.establish(mockConn)
warn.mockClear()
vi.useFakeTimers()
try {
const before = muxRequestMock.mock.calls.filter(
([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD
).length
muxRequestMock.mockRejectedValueOnce(Object.assign(new Error('controlled error'), { code }))
listener?.({ disabledTuiAgents: ['opencode2'] }, settings)
await vi.advanceTimersByTimeAsync(60_000)
expect(
muxRequestMock.mock.calls.filter(
([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD
)
).toHaveLength(before + 1)
expect(warn).toHaveBeenCalledTimes(code === 'PERMISSION_DENIED' ? 1 : 0)
} finally {
vi.useRealTimers()
warn.mockRestore()
session.dispose()
}
}
)
it.each([false, true])(
'bounds timeout retries and cancels them on disposal=%s',
async (dispose) => {
muxRequestMock.mockResolvedValue({ agents: [] })
const { mockStore, mockConn, mockPortForward, getMainWindow } = createMockDeps()
const settings = getDefaultSettings('/synthetic-home')
mockStore.getSettings = () => settings
let listener: Parameters<Store['onSettingsChanged']>[0] | undefined
mockStore.onSettingsChanged = (callback) => {
listener = callback
return vi.fn()
}
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const session = new SshRelaySession(
'target-bounded',
getMainWindow,
mockStore,
mockPortForward
)
await session.establish(mockConn)
const before = muxRequestMock.mock.calls.filter(
([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD
).length
muxRequestMock.mockImplementation(async (method) => {
if (method === AGENT_HOOK_INSTALL_PLUGINS_METHOD) {
throw Object.assign(new Error('timeout'), { code: 'SSH_MUX_REQUEST_TIMEOUT' })
}
return { agents: [] }
})
vi.useFakeTimers()
try {
listener?.({ disabledTuiAgents: ['opencode2'] }, settings)
if (dispose) {
session.dispose()
}
await vi.advanceTimersByTimeAsync(60_000)
expect(
muxRequestMock.mock.calls.filter(
([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD
)
).toHaveLength(before + (dispose ? 1 : 4))
} finally {
vi.useRealTimers()
warn.mockRestore()
session.dispose()
}
}
)
})
46 changes: 44 additions & 2 deletions src/main/ssh/ssh-relay-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ const SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS = 2
// store read, an attach round trip and a store write.
const SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS = 12
const SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS = 150
const SSH_PLUGIN_INSTALL_RETRY_DELAY_MS = 5_000
const SSH_PLUGIN_INSTALL_MAX_RETRIES = 3
const SSH_SOURCE_RECOVERY_CANCELLATION_FAILED = 'ssh_source_recovery_cancellation_failed'

// Why: superseded attempts stop quietly; a dead mux still owned by this attempt must enter recovery.
Expand Down Expand Up @@ -323,6 +325,8 @@ export class SshRelaySession {
// Why: hold the notification-handler disposer so teardownProviders can release it on reconnect/shutdown (symmetric with muxDisposeCleanup).
private muxNotificationCleanup: (() => void) | null = null
private pluginSettingsCleanup: (() => void) | null = null
private pluginInstallRetryTimer: ReturnType<typeof setTimeout> | null = null
private pluginInstallGeneration = 0
// Why: onStateChange never fires when the relay channel closes but SSH stays up; this callback lets ssh.ts drive relay-level reconnect.
private _onRelayLost: ((targetId: string) => void) | null = null
// Why: a version mismatch or a blocked owner admission is terminal, so it needs a separate callback
Expand Down Expand Up @@ -1611,11 +1615,16 @@ export class SshRelaySession {
}

// Why: ship plugin/extension source from Orca so agent-event changes don't force a relay redeploy — the relay is versioned independently. Best-effort: failure only costs agent status on this host.
private async installPluginsOnRelay(mux: SshChannelMultiplexer): Promise<void> {
private async installPluginsOnRelay(
mux: SshChannelMultiplexer,
attempt = 0,
generation = ++this.pluginInstallGeneration
): Promise<void> {
if (!isRemoteAgentHooksEnabled()) {
return
}
try {
this.clearPluginInstallRetry()
const hooksEnabled = this.areAgentStatusHooksEnabled()
await mux.request(
AGENT_HOOK_INSTALL_PLUGINS_METHOD,
Expand All @@ -1636,7 +1645,7 @@ export class SshRelaySession {
)
} catch (err) {
// Why: -32601 = older relay without the handler; CONNECTION_LOST/DISPOSED = routine mid-flight teardown — swallow both.
const code = (err as { code?: unknown })?.code
const code = err instanceof Error && 'code' in err ? err.code : undefined
if (code === -32601 || code === 'CONNECTION_LOST' || code === 'DISPOSED') {
return
}
Expand All @@ -1648,6 +1657,37 @@ export class SshRelaySession {
err instanceof Error ? err.message : String(err)
}`
)
if (code === 'SSH_MUX_REQUEST_TIMEOUT') {
this.schedulePluginInstallRetry(mux, attempt, generation)
}
}
}

private schedulePluginInstallRetry(
mux: SshChannelMultiplexer,
attempt: number,
generation: number
): void {
if (
attempt >= SSH_PLUGIN_INSTALL_MAX_RETRIES ||
this.mux !== mux ||
generation !== this.pluginInstallGeneration
) {
return
}
this.pluginInstallRetryTimer = setTimeout(() => {
this.pluginInstallRetryTimer = null
if (this.mux === mux && !mux.isDisposed()) {
void this.installPluginsOnRelay(mux, attempt + 1, generation)
}
}, SSH_PLUGIN_INSTALL_RETRY_DELAY_MS)
this.pluginInstallRetryTimer.unref?.()
}

private clearPluginInstallRetry(): void {
if (this.pluginInstallRetryTimer !== null) {
clearTimeout(this.pluginInstallRetryTimer)
this.pluginInstallRetryTimer = null
}
}

Expand Down Expand Up @@ -1763,6 +1803,8 @@ export class SshRelaySession {
this.releaseRelayLossWatcher()
this.pluginSettingsCleanup?.()
this.pluginSettingsCleanup = null
this.pluginInstallGeneration += 1
this.clearPluginInstallRetry()
this.leavePlainSshMode()
this.muxNotificationCleanup?.()
this.muxNotificationCleanup = null
Expand Down
Loading