Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,19 @@ vi.mock('node:os', async () => {
return { ...actual, homedir: () => testState.home }
})
// Prevent the selection-triggered history bridge from spawning a real app-server.

vi.mock('../codex/codex-account-session-index-heal', () => ({
createCodexAccountStateDb: async () => false,
healCodexAccountSessionIndex: async () => ({
vi.mock('../codex/codex-account-session-index-heal', () => {
const upToDate = async () => ({
outcome: 'up-to-date',
healedThreads: 0,
missingThreads: 0,
failedThreads: 0
})
}))
return {
createCodexAccountStateDb: async () => false,
healCodexAccountSessionIndex: upToDate,
healPendingCodexAccountThreads: upToDate
}
})

beforeEach(() => {
vi.resetModules()
Expand Down
4 changes: 4 additions & 0 deletions src/main/codex/codex-account-session-bridge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,10 @@ export function stopCodexAccountSessionBridges(): void {
stopping = true
}

export function isCodexAccountSessionBridgeStopping(): boolean {
return stopping
}

export async function bridgeCodexSessionsIntoAccountHome(args: {
targetCodexHomePath: string
sourceCodexHomePaths: readonly string[]
Expand Down
70 changes: 69 additions & 1 deletion src/main/codex/codex-account-session-index-heal.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ import { join } from 'node:path'
import {
_internals,
createCodexAccountStateDb,
healCodexAccountSessionIndex
healCodexAccountSessionIndex,
healPendingCodexAccountThreads
} from './codex-account-session-index-heal'
import {
CodexAppServerUnsupportedError,
Expand Down Expand Up @@ -302,3 +303,70 @@ describe('createCodexAccountStateDb', () => {
warn.mockRestore()
})
})

describe('healPendingCodexAccountThreads', () => {
it('reports each batch it settled, newest rollout first', async () => {
const { runSession } = fakeAppServer((threadId) => {
if (threadId === 'old') {
throw new Error('codex app-server thread/read failed: no rollout found for old')
}
})
const batches: string[][] = []

const summary = await healPendingCodexAccountThreads(
HOME,
new Map([
['old', '2026-07-01T10-00-00'],
['new', '2026-07-20T10-00-00']
]),
{
buildInvocation,
runSession,
readsPerServerSession: 1,
interBatchDelayMs: 0,
afterBatch: async (settled) => {
batches.push(settled.map(({ threadId, outcome }) => `${threadId}:${outcome}`))
}
}
)

expect(batches).toEqual([['new:healed'], ['old:missing']])
expect(summary).toMatchObject({ healedThreads: 1, missingThreads: 1 })
})

it('keeps indexing when post-batch bookkeeping fails', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { runSession, readThreadIds } = fakeAppServer()

const summary = await healPendingCodexAccountThreads(HOME, bridged('a', 'b'), {
buildInvocation,
runSession,
readsPerServerSession: 1,
interBatchDelayMs: 0,
afterBatch: async () => {
throw new Error('guest unreachable')
}
})

expect(readThreadIds.sort()).toEqual(['a', 'b'])
expect(summary.outcome).toBe('completed')
warn.mockRestore()
})
})

describe('createCodexAccountStateDb timeout', () => {
it('passes a caller timeout to the invocation', async () => {
const timeouts: number[] = []

await createCodexAccountStateDb(HOME, {
buildInvocation: (_home, timeoutMs) => {
timeouts.push(timeoutMs)
return buildInvocation()
},
runSession: async () => {},
timeoutMs: 900_000
})

expect(timeouts).toEqual([900_000])
})
})
41 changes: 28 additions & 13 deletions src/main/codex/codex-account-session-index-heal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,18 @@ const STATE_DB_CREATE_TIMEOUT_MS = 15_000
// Cache failed reads until restart to avoid respawning app-server for the same refused/missing thread.
const failedThreadIdsByHome = new Map<string, Set<string>>()

/** Initialize before bridging history to avoid startup backfill; false if Codex could not start. */
/** Starts Codex so the state DB exists and startup backfill is complete (app-server answers initialize only after both); false if Codex could not start. */
export async function createCodexAccountStateDb(
codexHomePath: string,
options: Pick<CodexSessionIndexHealOptions, 'buildInvocation' | 'runSession'> = {}
options: Pick<CodexSessionIndexHealOptions, 'buildInvocation' | 'runSession'> & {
timeoutMs?: number
} = {}
): Promise<boolean> {
const buildInvocation = options.buildInvocation ?? buildNativeHealInvocation
const runSession = options.runSession ?? runCodexAppServerSession
const timeoutMs = options.timeoutMs ?? STATE_DB_CREATE_TIMEOUT_MS
try {
await runSession(buildInvocation(codexHomePath, STATE_DB_CREATE_TIMEOUT_MS), async () => {})
await runSession(buildInvocation(codexHomePath, timeoutMs), async () => {})
return true
} catch (error) {
// Why: a Codex without app-server predates the state DB, so linking cannot stall it.
Expand All @@ -55,26 +58,34 @@ export async function healCodexAccountSessionIndex(
bridgedThreads: ReadonlyMap<string, string>,
options: CodexAccountSessionIndexHealOptions = {}
): Promise<CodexAccountSessionIndexHealSummary> {
const summary: CodexAccountSessionIndexHealSummary = {
outcome: 'up-to-date',
healedThreads: 0,
missingThreads: 0,
failedThreads: 0
}
if (bridgedThreads.size === 0) {
return summary
return upToDateSummary()
}
// No local DB means an older CLI or sqlite_home owns indexing elsewhere.
const indexed = (options.readIndexedThreadIds ?? readIndexedCodexThreadIds)(codexHomePath)
if (!indexed) {
return { ...summary, outcome: 'no-index' }
return { ...upToDateSummary(), outcome: 'no-index' }
}
const unindexed = new Map([...bridgedThreads].filter(([threadId]) => !indexed.has(threadId)))
return healPendingCodexAccountThreads(codexHomePath, unindexed, options)
}

/**
* Indexes `pendingThreads` (thread id -> rollout timestamp), newest rollout
* first, skipping threads that already failed in this home since Orca started.
*/
export async function healPendingCodexAccountThreads(
codexHomePath: string,
pendingThreads: ReadonlyMap<string, string>,
options: CodexSessionIndexHealOptions = {}
): Promise<CodexAccountSessionIndexHealSummary> {
const summary = upToDateSummary()
const homeKey = normalizeRuntimePathForComparison(codexHomePath)
const failed = failedThreadIdsByHome.get(homeKey) ?? new Set<string>()
failedThreadIdsByHome.set(homeKey, failed)
// /resume hides unindexed threads once any are indexed; prioritize recent work during long heals.
const pending = [...bridgedThreads]
.filter(([threadId]) => !indexed.has(threadId) && !failed.has(threadId))
const pending = [...pendingThreads]
.filter(([threadId]) => !failed.has(threadId))
.sort(([, left], [, right]) => (left < right ? 1 : left > right ? -1 : 0))
.map(([threadId]) => ({ threadId }))
if (pending.length === 0) {
Expand All @@ -100,6 +111,10 @@ export async function healCodexAccountSessionIndex(
return summary
}

function upToDateSummary(): CodexAccountSessionIndexHealSummary {
return { outcome: 'up-to-date', healedThreads: 0, missingThreads: 0, failedThreads: 0 }
}

export const _internals = {
resetFailedThreads: (): void => {
failedThreadIdsByHome.clear()
Expand Down
17 changes: 17 additions & 0 deletions src/main/codex/codex-session-index-heal.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { createCodexSessionBackfillAuditWriter } from './codex-session-backfill-
import { CODEX_SESSION_INDEX_HEAL_VERSION } from './codex-session-index-heal-state'
import {
buildNativeHealInvocation,
buildWslHealInvocation,
runCodexSessionIndexHeal,
type CodexSessionIndexHealPaths
} from './codex-session-index-heal'
Expand Down Expand Up @@ -799,3 +800,19 @@ describe('buildNativeHealInvocation', () => {
expect(invocation.env).toEqual({ CODEX_HOME: '/codex-home' })
})
})

describe('buildWslHealInvocation', () => {
it('runs the distro codex on the guest home with the short-lived read-only args', () => {
const invocation = buildWslHealInvocation('Ubuntu', '/home/alice/.orca-home', 2_000)

expect(invocation.cliPath).toBeNull()
expect(invocation.env).toBeUndefined()
expect(invocation.timeoutMs).toBe(2_000)
expect(invocation.command.toLowerCase()).toMatch(/wsl\.exe$/)
expect(invocation.args.slice(0, 4)).toEqual(['-d', 'Ubuntu', '--exec', 'sh'])
const guestCommand = invocation.args.at(-1) ?? ''
expect(guestCommand).toContain('/home/alice/.orca-home')
expect(guestCommand).toContain('features.plugins=false')
expect(guestCommand).toContain('app-server')
})
})
44 changes: 43 additions & 1 deletion src/main/codex/codex-session-index-heal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import { dirname, join } from 'node:path'
import { resolveCodexCommand } from '../codex-cli/command'
import { isTransientSqliteContention } from '../sqlite/sqlite-read-failure'
import { CODEX_SHORT_LIVED_PROBE_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args'
import { buildWslCodexAppServerArgs } from '../codex-accounts/wsl-codex-command'
import { resolveWslExecutablePath } from '../wsl/wsl-executable-path'
import { getCodexSessionBackfillStateDirPath } from './codex-home-paths'
import { resolveCodexSessionBackfillPaths } from './codex-session-backfill'
import {
Expand Down Expand Up @@ -51,8 +53,12 @@ export type CodexSessionIndexHealOptions = {
readsPerServerSession?: number
readConcurrency?: number
interBatchDelayMs?: number
/** Runs after each batch with the reads it settled; a failure is logged, not fatal. */
afterBatch?: (settled: readonly CodexSettledThreadRead[]) => Promise<void>
}

export type CodexSettledThreadRead = { threadId: string; outcome: HealLedgerOutcome }

export type CodexThreadReadOutcome = HealLedgerOutcome

export type CodexThreadReadPassOutcome = 'completed' | 'stopped' | 'unsupported' | 'aborted'
Expand Down Expand Up @@ -184,19 +190,23 @@ export async function readCodexThreadsForIndexHeal<T extends { threadId: string
}
const batch = threads.slice(offset, offset + readsPerServerSession)
const timeoutMs = HEAL_BATCH_TIMEOUT_BASE_MS + HEAL_BATCH_TIMEOUT_PER_READ_MS * batch.length
const settled: CodexSettledThreadRead[] = []
try {
await runSession(buildInvocation(codexHomePath, timeoutMs), async (rpc) => {
let nextIndex = 0
const worker = async (): Promise<void> => {
while (nextIndex < batch.length && !shouldStop()) {
const thread = batch[nextIndex]
nextIndex += 1
onOutcome(thread, await readOneThread(rpc, thread.threadId))
const outcome = await readOneThread(rpc, thread.threadId)
settled.push({ threadId: thread.threadId, outcome })
onOutcome(thread, outcome)
}
}
await Promise.all(Array.from({ length: readConcurrency }, () => worker()))
})
} catch (error) {
await runAfterBatch(options.afterBatch, settled)
if (shouldStop()) {
return 'stopped'
}
Expand All @@ -206,10 +216,25 @@ export async function readCodexThreadsForIndexHeal<T extends { threadId: string
console.warn('[codex-session-index-heal] Heal batch aborted:', error)
return 'aborted'
}
await runAfterBatch(options.afterBatch, settled)
}
return 'completed'
}

async function runAfterBatch(
afterBatch: CodexSessionIndexHealOptions['afterBatch'],
settled: readonly CodexSettledThreadRead[]
): Promise<void> {
if (!afterBatch || settled.length === 0) {
return
}
try {
await afterBatch(settled)
} catch (error) {
console.warn('[codex-session-index-heal] Post-batch bookkeeping failed:', error)
}
}

async function readOneThread(
rpc: CodexAppServerRpc,
threadId: string
Expand Down Expand Up @@ -269,3 +294,20 @@ export function buildNativeHealInvocation(
timeoutMs
}
}

/** Runs the distro's own codex against a guest home; Windows cannot open its sqlite over 9p. */
export function buildWslHealInvocation(
distro: string,
linuxCodexHomePath: string,
timeoutMs: number
): CodexAppServerInvocation {
return {
command: resolveWslExecutablePath(),
args: buildWslCodexAppServerArgs(distro, linuxCodexHomePath, [
...CODEX_SHORT_LIVED_PROBE_APP_SERVER_ARGS
]),
// Why null: the guest resolves `codex` inside the distro, so a host path pairs nothing.
cliPath: null,
timeoutMs
}
}
Loading
Loading