Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RemoteWorkspaceObservedSnapshot } from '../../../shared/remote-workspace-types'
import { createDeferredSnapshotPlacementRetries } from './remote-workspace-deferred-placement-retry'
import type { RemoteWorkspaceSnapshotPlacementStore } from './remote-workspace-snapshot-placement'
import {
appState,
flush,
owner,
snapshot
} from './__tests__/remote-workspace-target-sync-test-harness'

/** `appState()` carries worktree `repo-a::/remote/work`, so this path is placeable at once. */
const PLACEABLE_PATH = '/remote/work'

function placementStore(): RemoteWorkspaceSnapshotPlacementStore {
const state = appState()
return { getState: () => state, subscribe: () => () => {} }
}

const rejections: unknown[] = []
const onUnhandled = (reason: unknown): void => {
rejections.push(reason)
}
process.on('unhandledRejection', onUnhandled)
afterEach(() => {
rejections.length = 0
})

describe('createDeferredSnapshotPlacementRetries', () => {
it('swallows a getSnapshot rejection instead of leaving it unhandled', async () => {
const applySnapshot = vi.fn(async () => {})
const retries = createDeferredSnapshotPlacementRetries({
store: placementStore(),
getCurrentAuthority: () => owner,
getSnapshot: () => Promise.reject(new Error('relay dropped')),
applySnapshot
})

retries.watch(owner, [PLACEABLE_PATH])
await flush()
await new Promise((resolve) => setTimeout(resolve, 0))

expect(rejections).toEqual([])
// A failed pull is unverifiable, so the target is left on `conflict` rather than re-applied.
expect(applySnapshot).not.toHaveBeenCalled()
retries.stop()
})

it('swallows an applySnapshot rejection instead of leaving it unhandled', async () => {
const retries = createDeferredSnapshotPlacementRetries({
store: placementStore(),
getCurrentAuthority: () => owner,
getSnapshot: async () => snapshot(4),
applySnapshot: () => Promise.reject(new Error('relay dropped'))
})

retries.watch(owner, [PLACEABLE_PATH])
await flush()
await new Promise((resolve) => setTimeout(resolve, 0))

expect(rejections).toEqual([])
retries.stop()
})

it('bounds an apply that keeps re-arming the same unplaced paths', async () => {
// Guards the test itself: without a bound the chain never yields and the run would hang.
const CHAIN_GUARD = 50
let pulls = 0
const retries = createDeferredSnapshotPlacementRetries({
store: placementStore(),
getCurrentAuthority: () => owner,
getSnapshot: async (): Promise<RemoteWorkspaceObservedSnapshot> => {
pulls += 1
return snapshot(4 + pulls)
},
applySnapshot: async () => {
// The apply reports the same still-unplaced path, which arms the next watch.
if (pulls < CHAIN_GUARD) {
retries.watch(owner, [PLACEABLE_PATH])
}
}
})

retries.watch(owner, [PLACEABLE_PATH])
for (let tick = 0; tick < CHAIN_GUARD * 2; tick += 1) {
await flush()
}

expect(pulls).toBeLessThan(CHAIN_GUARD)
expect(pulls).toBe(3)
retries.stop()
})

it('lets a chain that keeps placing rows run past the stalled bound', async () => {
// Five placeable paths, one fewer unplaced each round: that chain is converging and is already
// bounded by the set emptying, so the stall bound must not cut it off at 3.
const state = appState({
worktreesByRepo: {
'repo-a': [1, 2, 3, 4, 5].map((n) => ({
id: `repo-a::/remote/w${n}`,
repoId: 'repo-a',
hostId: 'ssh:target-a'
}))
}
})
const allPaths = [1, 2, 3, 4, 5].map((n) => `/remote/w${n}`)
let pulls = 0
let remaining = allPaths.length
const retries = createDeferredSnapshotPlacementRetries({
store: { getState: () => state, subscribe: () => () => {} },
getCurrentAuthority: () => owner,
getSnapshot: async (): Promise<RemoteWorkspaceObservedSnapshot> => {
pulls += 1
return snapshot(4 + pulls)
},
applySnapshot: async () => {
remaining -= 1
retries.watch(owner, allPaths.slice(0, remaining))
}
})

retries.watch(owner, allPaths)
for (let tick = 0; tick < 40; tick += 1) {
await flush()
}

// One pull per round until the unplaced set empties -- five, not the stall bound of three.
expect(pulls).toBe(5)
retries.stop()
})

it('does not spend the chain budget on watches armed outside a retry', async () => {
let pulls = 0
const retries = createDeferredSnapshotPlacementRetries({
store: placementStore(),
getCurrentAuthority: () => owner,
getSnapshot: async (): Promise<RemoteWorkspaceObservedSnapshot> => {
pulls += 1
return snapshot(4 + pulls)
},
applySnapshot: async () => {}
})

// Each of these is a fresh host snapshot arrival, not a continued chain.
for (let arrival = 0; arrival < 6; arrival += 1) {
retries.watch(owner, [PLACEABLE_PATH])
await flush()
await flush()
}

expect(pulls).toBe(6)
retries.stop()
})
})
129 changes: 129 additions & 0 deletions src/renderer/src/hooks/remote-workspace-deferred-placement-retry.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
import type { DirectSshAuthority } from '../../../shared/ssh-types'
import type { RemoteWorkspaceObservedSnapshot } from '../../../shared/remote-workspace-types'
import { directSshAuthoritiesEqual } from './direct-ssh-reconnect-tokens'
import {
waitForSnapshotWorktreePlacement,
type RemoteWorkspaceSnapshotPlacementStore
} from './remote-workspace-snapshot-placement'

/** How long a conflicted target keeps watching the catalog for the rows it could not place. The
* catalog for a remote host often only fills in when the user opens the worktree, which is minutes
* after connect, and `conflict` has no other exit: it suppresses uploads and holds terminal
* authority at `unverifiable` until a reconnect or an unsolicited host push happens to arrive. */
const DEFERRED_SNAPSHOT_PLACEMENT_TIMEOUT_MS = 600_000

/** A retry's own apply can report paths it still could not place, which arms the next watch. If the
* catalog reports those paths placeable, that watch fires at once and the chain never yields.
*
* Counted only while the unplaced set stops shrinking: a chain that keeps placing rows is
* converging and is already bounded by that set emptying, so cutting it short would strand the
* target on `conflict` for no reason. A chain that re-arms on the same or a larger set is the
* spinning case, and that is what this bounds. */
const MAX_STALLED_DEFERRED_PLACEMENT_CHAIN = 3

export type DeferredSnapshotPlacementRetryDeps = {
store: RemoteWorkspaceSnapshotPlacementStore
getCurrentAuthority: (targetId: string) => DirectSshAuthority | null
getSnapshot: (targetId: string) => Promise<RemoteWorkspaceObservedSnapshot | null>
applySnapshot: (targetId: string, snapshot: RemoteWorkspaceObservedSnapshot) => Promise<void>
}

export type DeferredSnapshotPlacementRetries = {
/** Empty `worktreePaths` retires the target's outstanding watch instead of arming one. */
watch: (authority: DirectSshAuthority, worktreePaths: readonly string[]) => void
stop: () => void
}

/**
* Re-pull once the local catalog can place the host paths an apply had to drop.
*
* Why a fresh pull rather than replaying the snapshot already held: that snapshot is only evidence
* of what the host had when it was taken. The host owns execution state, so the answer acted on has
* to be its current one — see docs/reference/ssh-execution-boundary.md. A pull that comes back
* empty or fails is left alone: it is `unverifiable`, and moving the target off `conflict` on it
* would re-authorise uploads and sleeping-agent resume from a picture known to be incomplete.
*/
export function createDeferredSnapshotPlacementRetries(
deps: DeferredSnapshotPlacementRetryDeps
): DeferredSnapshotPlacementRetries {
const watchers = new Map<string, AbortController>()
/** Consecutive non-shrinking re-arms, and the set size they stalled at. Absent means no chain. */
const chains = new Map<string, { stalledDepth: number; unplacedCount: number }>()
const applying = new Set<string>()
let stopped = false

const watch = (authority: DirectSshAuthority, worktreePaths: readonly string[]): void => {
// Always retire the previous watch: an apply that placed everything passes no paths, and that
// is exactly when the outstanding watch is obsolete.
watchers.get(authority.targetId)?.abort()
watchers.delete(authority.targetId)
// A watch armed from outside a retry is a fresh snapshot arrival, not a continued chain.
const previous = applying.has(authority.targetId) ? chains.get(authority.targetId) : undefined

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | 🏗️ Heavy lift

Keep the retry budget separate from fresh snapshot arrivals.

applying.has(authority.targetId) only shows that an apply is pending. It does not show that this watch call came from that apply. If a fresh host snapshot arrives while an earlier applySnapshot is pending for the same target, this branch reuses the old stalledDepth. At depth two with the same path count, line 68 discards the fresh watch. The target then stays in conflict without a recovery attempt.

Split fresh watch calls from internal retry re-arms, or pass explicit retry provenance. Add a test for a fresh arrival that overlaps a stalled apply.

if (stopped || worktreePaths.length === 0) {
chains.delete(authority.targetId)
return
}
const stalledDepth =
previous && worktreePaths.length >= previous.unplacedCount ? previous.stalledDepth + 1 : 0
if (stalledDepth >= MAX_STALLED_DEFERRED_PLACEMENT_CHAIN) {
// Leave the target on `conflict`: the paths are not becoming placeable by re-pulling.
chains.delete(authority.targetId)
return
}
chains.set(authority.targetId, { stalledDepth, unplacedCount: worktreePaths.length })
const controller = new AbortController()
watchers.set(authority.targetId, controller)
const isCurrent = (): boolean =>
!stopped &&
watchers.get(authority.targetId) === controller &&
directSshAuthoritiesEqual(deps.getCurrentAuthority(authority.targetId), authority)
void (async () => {
Comment thread
pullfrog[bot] marked this conversation as resolved.
Resolved
try {
const placed = await waitForSnapshotWorktreePlacement(
deps.store,
authority,
worktreePaths,
isCurrent,
controller.signal,
DEFERRED_SNAPSHOT_PLACEMENT_TIMEOUT_MS
)
if (!placed || !isCurrent()) {
return
}
const snapshot = await deps.getSnapshot(authority.targetId)
if (!snapshot || snapshot.revision <= 0 || !isCurrent()) {
return
}
applying.add(authority.targetId)
try {
await deps.applySnapshot(authority.targetId, snapshot)
} finally {
applying.delete(authority.targetId)
}
} catch {
// `getSnapshot` is an IPC call that rejects when the relay drops, and the apply can reject
// with it. A failed pull is `unverifiable`, and the documented behaviour is to leave the
// target on `conflict` rather than act on a picture known to be incomplete -- so swallow it
// here instead of surfacing an unhandled rejection.
} finally {
// A still-current controller means the apply did not re-arm, so the chain ends here.
if (watchers.get(authority.targetId) === controller) {
watchers.delete(authority.targetId)
chains.delete(authority.targetId)
}
}
})()
}

return {
watch,
stop: () => {
stopped = true
for (const controller of watchers.values()) {
controller.abort()
}
watchers.clear()
chains.clear()
}
}
}
10 changes: 9 additions & 1 deletion src/renderer/src/hooks/remote-workspace-snapshot-apply.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,12 @@ type RemoteWorkspaceSnapshotApplyInput = {
isPreparationTokenCurrent: (token: DirectSshPreparationToken) => boolean
waitForWorkspaceSessionReady: (signal?: AbortSignal) => Promise<boolean>
finalizeHydratedTerminals: (authority: DirectSshAuthority) => number
/**
* Host paths still carrying terminal tabs when this apply gave up placing them. `unverifiable`,
* never proof the rows are not ours, so the caller owns getting back to a placed picture — this
* apply itself has no way back once the bounded in-apply wait expires.
*/
onUnplacedTabWorktreePaths?: (worktreePaths: readonly string[]) => void
}

export type RemoteWorkspaceSnapshotApplyResult = 'applied' | 'stale' | 'failed'
Expand Down Expand Up @@ -115,7 +121,8 @@ export async function applyDirectSshRemoteWorkspaceSnapshot({
isArrivalCurrent,
isPreparationTokenCurrent,
waitForWorkspaceSessionReady,
finalizeHydratedTerminals
finalizeHydratedTerminals,
onUnplacedTabWorktreePaths
}: RemoteWorkspaceSnapshotApplyInput): Promise<RemoteWorkspaceSnapshotApplyResult> {
const { authority } = token
if (!isArrivalCurrent(authority.targetId, arrival)) {
Expand Down Expand Up @@ -181,6 +188,7 @@ export async function applyDirectSshRemoteWorkspaceSnapshot({
return 'stale'
}
const hasUnplacedTerminalTabs = unplacedTabWorktreePaths.length > 0
onUnplacedTabWorktreePaths?.([...unplacedTabWorktreePaths])
snapshotApplyDepth += 1
try {
const currentStore = store.getState()
Expand Down
22 changes: 20 additions & 2 deletions src/renderer/src/hooks/remote-workspace-snapshot-placement.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,23 @@ export function resolveDirectSshSnapshotWorktreeIds(
return worktreeIds
}

/** Only the rows the snapshot is allowed to replace, with no host-qualified widening. */
export function resolveExactDirectSshTargetWorktreeIds(
state: AppState,
authority: DirectSshAuthority
): Set<string> {
return resolveDirectSshTargetScope({
targetId: authority.targetId,
catalogRevision: 0,
repos: state.repos,
worktreesByRepo: state.worktreesByRepo,
detectedWorktreesByRepo: state.detectedWorktreesByRepo,
folderWorkspaces: state.folderWorkspaces,
projectGroups: state.projectGroups,
restoredRuntimeHostIdByWorkspaceSessionKey: state.restoredRuntimeHostIdByWorkspaceSessionKey
}).gitWorktreeIds
}

function snapshotPathsArePlaceable(
state: AppState,
authority: DirectSshAuthority,
Expand Down Expand Up @@ -85,7 +102,8 @@ export async function waitForSnapshotWorktreePlacement(
authority: DirectSshAuthority,
worktreePaths: readonly string[],
isCurrent: () => boolean,
signal?: AbortSignal
signal?: AbortSignal,
timeoutMs: number = SNAPSHOT_WORKTREE_PLACEMENT_TIMEOUT_MS
): Promise<boolean> {
if (signal?.aborted || !isCurrent()) {
return false
Expand Down Expand Up @@ -117,7 +135,7 @@ export async function waitForSnapshotWorktreePlacement(
resolve(placed)
}
const onAbort = (): void => finish(false)
timer = setTimeout(() => finish(false), SNAPSHOT_WORKTREE_PLACEMENT_TIMEOUT_MS)
timer = setTimeout(() => finish(false), timeoutMs)
signal?.addEventListener('abort', onAbort, { once: true })
const subscribedUnsubscribe = store.subscribe((state) => {
if (!isCurrent()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
*
* The oracle is the promotion, not the drop. Without a local catalog there is nowhere to put the
* rows, and that is fine and recoverable — what is not recoverable is declaring the empty result
* authoritative, because nothing re-pulls after the lineage lands. An unplaceable row is
* `unverifiable`, never `exited` (docs/reference/ssh-execution-boundary.md).
* authoritative. An unplaceable row is `unverifiable`, never `exited`
* (docs/reference/ssh-execution-boundary.md). Getting back to a placed picture is the caller's job:
* this apply has no way back once its bounded wait expires, so it reports the paths it dropped and
* remote-workspace-target-sync.ts re-pulls when the catalog can place them.
*
* The catalog-present case is pinned alongside it so the gate cannot be satisfied by never
* hydrating anything.
Expand Down Expand Up @@ -275,7 +277,8 @@ describe('a host snapshot whose terminal tabs cannot be placed locally', () => {
expect(adoptedTabIds(store), 'no local worktree row exists to hang the host tabs on').toEqual(
[]
)
// Not recoverable: promoting that to truth. Nothing re-pulls once the lineage lands.
// Not recoverable: promoting that to truth. This apply never re-pulls on its own; the deferred
// placement watch in remote-workspace-target-sync.ts is what re-pulls once the lineage lands.
expect(
isHydrated(store),
'the host named 3 terminals and this client placed none of them, so the target is not hydrated'
Expand Down Expand Up @@ -335,7 +338,7 @@ describe('a host snapshot whose terminal tabs cannot be placed locally', () => {
const store = createStore()

// First pass: the lineage read was degraded, so nothing places and the target is left
// un-hydrated on purpose. With the retry chain gone, this is the only way back.
// un-hydrated on purpose. A later snapshot is how this apply, on its own, gets back.
await applySnapshot(store, snapshot(1))
expect(adoptedTabIds(store)).toEqual([])
expect(isHydrated(store)).toBe(false)
Expand Down
Loading