From dd90c5bff7eddf09bb5c4f616c6ef375f9600284 Mon Sep 17 00:00:00 2001 From: Justas Brazauskas Date: Fri, 2 Oct 2026 22:09:57 -0700 Subject: [PATCH] fix(opencode): retry timed-out SSH plugin updates Preserve bounded retry behavior and the current-main status-envelope fields. Original-PR: #24124 Reviewed-source: 103144f9c5029f9217f64970271a5b7f69d3f823 --- .../ssh-relay-session-managed-hooks.test.ts | 186 ++++++++++++++++++ src/main/ssh/ssh-relay-session.ts | 46 ++++- 2 files changed, 230 insertions(+), 2 deletions(-) diff --git a/src/main/ssh/ssh-relay-session-managed-hooks.test.ts b/src/main/ssh/ssh-relay-session-managed-hooks.test.ts index da10d8841a18..1ef197e170d0 100644 --- a/src/main/ssh/ssh-relay-session-managed-hooks.test.ts +++ b/src/main/ssh/ssh-relay-session-managed-hooks.test.ts @@ -202,4 +202,190 @@ describe('SshRelaySession managed hooks', () => { session.dispose() expect(cleanup).toHaveBeenCalledOnce() }) + + // Why: a timed-out install left the relay on the old sources (a disabled agent kept its plugin) until the next reconnect. + it('retries a settings-triggered plugin install that failed, using the latest settings', async () => { + muxRequestMock.mockResolvedValue({ agents: [] }) + const { mockStore, mockConn, mockPortForward, getMainWindow } = createMockDeps() + const settings = getDefaultSettings('/synthetic-home') + mockStore.getSettings = () => settings + let listener: Parameters[0] | undefined + mockStore.onSettingsChanged = (callback) => { + listener = callback + return vi.fn() + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const session = new SshRelaySession( + 'target-settings', + getMainWindow, + mockStore, + mockPortForward + ) + await session.establish(mockConn) + const installCalls = () => + muxRequestMock.mock.calls.filter(([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD) + + vi.useFakeTimers() + try { + const before = installCalls().length + muxRequestMock.mockImplementationOnce(async (method: string) => { + if (method !== AGENT_HOOK_INSTALL_PLUGINS_METHOD) { + return { agents: [] } + } + throw Object.assign(new Error('timed out'), { code: 'SSH_MUX_REQUEST_TIMEOUT' }) + }) + settings.disabledTuiAgents = ['opencode2'] + listener?.({ disabledTuiAgents: settings.disabledTuiAgents }, settings) + await vi.advanceTimersByTimeAsync(60_000) + + const calls = installCalls().slice(before) + expect(calls.length).toBeGreaterThan(1) + expect(calls.at(-1)?.[1]).toMatchObject({ opencode2PluginSource: '' }) + } finally { + vi.useRealTimers() + warn.mockRestore() + session.dispose() + } + }) + + // Why: two overlapping failed installs must leave one retry chain, not two the next install can't cancel. + it('retries once when overlapping settings-triggered installs both fail', async () => { + muxRequestMock.mockResolvedValue({ agents: [] }) + const { mockStore, mockConn, mockPortForward, getMainWindow } = createMockDeps() + const settings = getDefaultSettings('/synthetic-home') + mockStore.getSettings = () => settings + let listener: Parameters[0] | undefined + mockStore.onSettingsChanged = (callback) => { + listener = callback + return vi.fn() + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const session = new SshRelaySession( + 'target-settings', + getMainWindow, + mockStore, + mockPortForward + ) + await session.establish(mockConn) + const installCalls = () => + muxRequestMock.mock.calls.filter(([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD) + + vi.useFakeTimers() + try { + const before = installCalls().length + const pendingInstalls: ((error: Error) => void)[] = [] + muxRequestMock.mockImplementation((method: string) => { + if (method === AGENT_HOOK_INSTALL_PLUGINS_METHOD && pendingInstalls.length < 2) { + return new Promise((_resolve, reject) => pendingInstalls.push(reject)) + } + return Promise.resolve({ agents: [] }) + }) + settings.disabledTuiAgents = ['opencode2'] + listener?.({ disabledTuiAgents: settings.disabledTuiAgents }, settings) + settings.disabledTuiAgents = ['opencode'] + listener?.({ disabledTuiAgents: settings.disabledTuiAgents }, settings) + expect(pendingInstalls).toHaveLength(2) + + for (const reject of pendingInstalls) { + reject(Object.assign(new Error('timed out'), { code: 'SSH_MUX_REQUEST_TIMEOUT' })) + } + await vi.advanceTimersByTimeAsync(60_000) + + expect(installCalls().slice(before)).toHaveLength(3) + } finally { + vi.useRealTimers() + warn.mockRestore() + session.dispose() + } + }) + it.each(['PERMISSION_DENIED', -32601, 'CONNECTION_LOST', 'DISPOSED'])( + 'does not retry permanent or retired transport failure %s', + async (code) => { + muxRequestMock.mockResolvedValue({ agents: [] }) + const { mockStore, mockConn, mockPortForward, getMainWindow } = createMockDeps() + const settings = getDefaultSettings('/synthetic-home') + mockStore.getSettings = () => settings + let listener: Parameters[0] | undefined + mockStore.onSettingsChanged = (callback) => { + listener = callback + return vi.fn() + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const session = new SshRelaySession( + 'target-no-retry', + getMainWindow, + mockStore, + mockPortForward + ) + await session.establish(mockConn) + warn.mockClear() + vi.useFakeTimers() + try { + const before = muxRequestMock.mock.calls.filter( + ([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD + ).length + muxRequestMock.mockRejectedValueOnce(Object.assign(new Error('controlled error'), { code })) + listener?.({ disabledTuiAgents: ['opencode2'] }, settings) + await vi.advanceTimersByTimeAsync(60_000) + expect( + muxRequestMock.mock.calls.filter( + ([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD + ) + ).toHaveLength(before + 1) + expect(warn).toHaveBeenCalledTimes(code === 'PERMISSION_DENIED' ? 1 : 0) + } finally { + vi.useRealTimers() + warn.mockRestore() + session.dispose() + } + } + ) + it.each([false, true])( + 'bounds timeout retries and cancels them on disposal=%s', + async (dispose) => { + muxRequestMock.mockResolvedValue({ agents: [] }) + const { mockStore, mockConn, mockPortForward, getMainWindow } = createMockDeps() + const settings = getDefaultSettings('/synthetic-home') + mockStore.getSettings = () => settings + let listener: Parameters[0] | undefined + mockStore.onSettingsChanged = (callback) => { + listener = callback + return vi.fn() + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const session = new SshRelaySession( + 'target-bounded', + getMainWindow, + mockStore, + mockPortForward + ) + await session.establish(mockConn) + const before = muxRequestMock.mock.calls.filter( + ([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD + ).length + muxRequestMock.mockImplementation(async (method) => { + if (method === AGENT_HOOK_INSTALL_PLUGINS_METHOD) { + throw Object.assign(new Error('timeout'), { code: 'SSH_MUX_REQUEST_TIMEOUT' }) + } + return { agents: [] } + }) + vi.useFakeTimers() + try { + listener?.({ disabledTuiAgents: ['opencode2'] }, settings) + if (dispose) { + session.dispose() + } + await vi.advanceTimersByTimeAsync(60_000) + expect( + muxRequestMock.mock.calls.filter( + ([method]) => method === AGENT_HOOK_INSTALL_PLUGINS_METHOD + ) + ).toHaveLength(before + (dispose ? 1 : 4)) + } finally { + vi.useRealTimers() + warn.mockRestore() + session.dispose() + } + } + ) }) diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index 673fc25fed58..2cd1f394cd72 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -162,6 +162,8 @@ const SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS = 2 // store read, an attach round trip and a store write. const SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS = 12 const SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS = 150 +const SSH_PLUGIN_INSTALL_RETRY_DELAY_MS = 5_000 +const SSH_PLUGIN_INSTALL_MAX_RETRIES = 3 const SSH_SOURCE_RECOVERY_CANCELLATION_FAILED = 'ssh_source_recovery_cancellation_failed' // Why: superseded attempts stop quietly; a dead mux still owned by this attempt must enter recovery. @@ -323,6 +325,8 @@ export class SshRelaySession { // Why: hold the notification-handler disposer so teardownProviders can release it on reconnect/shutdown (symmetric with muxDisposeCleanup). private muxNotificationCleanup: (() => void) | null = null private pluginSettingsCleanup: (() => void) | null = null + private pluginInstallRetryTimer: ReturnType | null = null + private pluginInstallGeneration = 0 // Why: onStateChange never fires when the relay channel closes but SSH stays up; this callback lets ssh.ts drive relay-level reconnect. private _onRelayLost: ((targetId: string) => void) | null = null // Why: a version mismatch or a blocked owner admission is terminal, so it needs a separate callback @@ -1611,11 +1615,16 @@ export class SshRelaySession { } // Why: ship plugin/extension source from Orca so agent-event changes don't force a relay redeploy — the relay is versioned independently. Best-effort: failure only costs agent status on this host. - private async installPluginsOnRelay(mux: SshChannelMultiplexer): Promise { + private async installPluginsOnRelay( + mux: SshChannelMultiplexer, + attempt = 0, + generation = ++this.pluginInstallGeneration + ): Promise { if (!isRemoteAgentHooksEnabled()) { return } try { + this.clearPluginInstallRetry() const hooksEnabled = this.areAgentStatusHooksEnabled() await mux.request( AGENT_HOOK_INSTALL_PLUGINS_METHOD, @@ -1636,7 +1645,7 @@ export class SshRelaySession { ) } catch (err) { // Why: -32601 = older relay without the handler; CONNECTION_LOST/DISPOSED = routine mid-flight teardown — swallow both. - const code = (err as { code?: unknown })?.code + const code = err instanceof Error && 'code' in err ? err.code : undefined if (code === -32601 || code === 'CONNECTION_LOST' || code === 'DISPOSED') { return } @@ -1648,6 +1657,37 @@ export class SshRelaySession { err instanceof Error ? err.message : String(err) }` ) + if (code === 'SSH_MUX_REQUEST_TIMEOUT') { + this.schedulePluginInstallRetry(mux, attempt, generation) + } + } + } + + private schedulePluginInstallRetry( + mux: SshChannelMultiplexer, + attempt: number, + generation: number + ): void { + if ( + attempt >= SSH_PLUGIN_INSTALL_MAX_RETRIES || + this.mux !== mux || + generation !== this.pluginInstallGeneration + ) { + return + } + this.pluginInstallRetryTimer = setTimeout(() => { + this.pluginInstallRetryTimer = null + if (this.mux === mux && !mux.isDisposed()) { + void this.installPluginsOnRelay(mux, attempt + 1, generation) + } + }, SSH_PLUGIN_INSTALL_RETRY_DELAY_MS) + this.pluginInstallRetryTimer.unref?.() + } + + private clearPluginInstallRetry(): void { + if (this.pluginInstallRetryTimer !== null) { + clearTimeout(this.pluginInstallRetryTimer) + this.pluginInstallRetryTimer = null } } @@ -1763,6 +1803,8 @@ export class SshRelaySession { this.releaseRelayLossWatcher() this.pluginSettingsCleanup?.() this.pluginSettingsCleanup = null + this.pluginInstallGeneration += 1 + this.clearPluginInstallRetry() this.leavePlainSshMode() this.muxNotificationCleanup?.() this.muxNotificationCleanup = null