diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index e58c8a1e528f..e0ea6779370d 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -1712,6 +1712,126 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( }, ); +it.effect.each(["manual", "mcp", "stop"] as const)( + "holds a recovered child's parent wake until %s without losing the result", + (action) => { + return Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + using offered = vi.spyOn(requests, "offer"); + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread:recovery-wake:${action}`); + const projectId = ProjectId.make(`project:recovery-wake:${action}`); + const runId = RunId.make(`run:recovery-wake:${action}`); + const rootNodeId = NodeId.make(`node:recovery-wake:${action}`); + yield* seedParentWithTerminalTask({ + threadId, + projectId, + runId, + rootNodeId, + taskId: NodeId.make(`node:recovery-wake:seed:${action}`), + deliveryState: "acknowledged", + now, + }); + const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; + yield* sink.write({ + commandId: reconcileCommandId(`recovery-wake:${action}`), + events: [ + { + id: EventId.make(`event:recovery-wake:cut:${action}`), + type: "run.updated", + threadId, + occurredAt: now, + payload: { ...parent, status: "cancelled", completedAt: now }, + }, + ], + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name: `recovery-wake-child:${action}`, + completionWake: "always", + continuationPending: true, + now, + }); + yield* continueRestartedRun({ + threadId: child.childThreadId, + sourceRunId: child.childRunId, + }).pipe( + Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false })), + ); + yield* orchestrator.recoverDelegatedTasks; + const recovered = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + recovered.subagents.find((task) => task.id === child.taskId)?.status, + "cancelled", + ); + assert.isNotNull(recovered.runs[0]?.delegatedCompletion?.delivery); + const delivery = recovered.runs[0]!.delegatedCompletion!.delivery!; + const wakeCommand = { + type: "message.dispatch" as const, + threadId, + commandId: CommandId.make(`command:recovery-wake:delivery:${action}`), + messageId: delivery.messageId, + text: "Child cancelled", + attachments: [], + dispatchMode: { type: "queue_after_active" as const }, + createdBy: "agent" as const, + creationSource: "server" as const, + delegatedCompletion: { + parentRunId: runId, + generation: delivery.generation, + taskIds: delivery.taskIds, + }, + }; + yield* orchestrator.dispatch(wakeCommand); + yield* orchestrator.dispatch({ + ...wakeCommand, + commandId: CommandId.make(`command:recovery-wake:repeated:${action}`), + }); + assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1); + assert.equal(offered.mock.calls.length, 0); + if (action !== "stop") { + yield* orchestrator.dispatch({ + type: "message.dispatch", + threadId, + commandId: CommandId.make("command:recovery-wake:manual"), + messageId: MessageId.make("message:recovery-wake:manual"), + text: "Continue", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: action === "mcp" ? "agent" : "user", + creationSource: action === "mcp" ? "mcp" : "web", + }); + assert.equal(offered.mock.calls.length, 1); + const request = yield* requests.take; + assert.equal(request.delegatedCompletion?.messageId, delivery.messageId); + yield* orchestrator.dispatch({ + ...wakeCommand, + commandId: CommandId.make("command:recovery-wake:retry"), + }); + const resumed = yield* orchestrator.getThreadProjection(threadId); + assert.lengthOf(resumed.runs, 3); + assert.equal( + resumed.runs.find((run) => run.userMessageId === delivery.messageId)?.status, + "queued", + ); + } else { + yield* orchestrator.dispatch({ + type: "thread.stop", + threadId, + commandId: CommandId.make("command:recovery-wake:stop"), + }); + yield* orchestrator.recoverDelegatedTasks; + assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1); + } + }).pipe(Effect.provide(Layer.merge(layerTest, ProviderContinuationRequests.layer))); + }, +); + it.layer(layerTest)("delegated tasks across a server restart", (it) => { it.effect("queues a child result without creating a provider history handoff", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 6818ba361df3..30f7740f52a4 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -33,6 +33,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ Schema.Struct({ type: Schema.Literal("provider-runtime.continue"), sourceRunId: RunId, + processLossCount: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))), }), Schema.Struct({ type: Schema.Literal("provider-session.detach"), @@ -321,6 +322,8 @@ export const layer: Layer.Layer = La ).pipe(Effect.asVoid); // An earlier effect waiting out a retry backoff still blocks later ones, so // a turn cannot start while a failed rollback is about to restore files. + // Dispatch finishes before the turn does. Keep admission through that run, + // and recover children first so a waiting parent cannot keep them out. const claimableCandidatePredicate = ( availableBefore?: string, excludeRestartContinuations = false, @@ -332,6 +335,31 @@ export const layer: Layer.Layer = La : sql`candidate.available_at <= ${availableBefore}` } AND candidate.status = 'pending' + AND ( + candidate.effect_type != 'provider-runtime.continue' + OR ( + NOT EXISTS ( + SELECT 1 FROM orchestration_v2_effect_outbox AS recovery + WHERE recovery.effect_type = 'provider-runtime.continue' + AND recovery.status = 'running' + ) + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runs AS recovery_run + WHERE recovery_run.status IN ('starting', 'running', 'waiting') + AND json_extract(recovery_run.payload_json, '$.restartContinuationOfRunId') IS NOT NULL + ) + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_subagents AS child + JOIN orchestration_v2_effect_outbox AS child_recovery + ON child_recovery.thread_id = child.child_thread_id + WHERE child.thread_id = candidate.thread_id + AND child.origin = 'app_owned' + AND child.status IN ('pending', 'running', 'waiting') + AND child_recovery.effect_type = 'provider-runtime.continue' + AND child_recovery.status IN ('pending', 'running') + ) + ) + ) AND NOT EXISTS ( SELECT 1 FROM orchestration_v2_effect_outbox AS active @@ -344,7 +372,7 @@ export const layer: Layer.Layer = La AND ${ excludeRestartContinuations ? sql`active.effect_type != 'provider-runtime.continue'` - : sql`1 = 1` + : sql`(active.effect_type != 'provider-runtime.continue' OR candidate.effect_type = 'provider-runtime.continue')` } ) ) @@ -693,6 +721,8 @@ export const layer: Layer.Layer = La const cutoff = DateTime.formatIso( DateTime.subtractDuration(yield* DateTime.now, SETTLED_EFFECT_RETENTION), ); + // A live continuation still needs its intent's crash budget on its next + // restart, even if that turn runs longer than the normal retention. // Each batch is its own statement, so a large backlog never holds the // write lock for long and writers can commit between batches. const deleteBatch = sql<{ readonly effect_id: string }>` @@ -702,6 +732,16 @@ export const layer: Layer.Layer = La FROM orchestration_v2_effect_outbox WHERE status IN ('succeeded', 'cancelled') AND completed_at < ${cutoff} + AND NOT ( + effect_type = 'provider-runtime.continue' + AND EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runs AS recovery_run + WHERE recovery_run.status IN ('starting', 'running', 'waiting') + AND json_extract(recovery_run.payload_json, '$.restartContinuationOfRunId') IS NOT NULL + AND json_extract(recovery_run.payload_json, '$.restartContinuationOfRunId') = + json_extract(orchestration_v2_effect_outbox.payload_json, '$.sourceRunId') + ) + ) LIMIT ${PRUNE_BATCH_SIZE} ) RETURNING effect_id diff --git a/apps/server/src/orchestration-v2/EffectWorker.test.ts b/apps/server/src/orchestration-v2/EffectWorker.test.ts index 671d7902fb45..816712ba8e03 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.test.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.test.ts @@ -4,6 +4,8 @@ import { CheckpointScopeId, CommandId, NodeId, + ProjectId, + ProviderInstanceId, ProviderSessionId, ProviderThreadId, ProviderTurnId, @@ -11,6 +13,7 @@ import { RunId, ThreadId, VcsProcessExitError, + type OrchestrationV2ThreadProjection, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; @@ -475,6 +478,62 @@ it.effect("terminalizes a process-bound claim when success settlement fails", () }), ); +it.effect.each([0, 3, 4])( + "enforces process-loss count %s after 22 minutes through the executor", + (processLossCount) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const instanceId = ProviderInstanceId.make("codex"); + const events = yield* Ref.make>([]); + const layer = layerExecutorFor({ + events, + continueAfterRestart: true, + threads: { + getThreadRecords: () => + Effect.succeed({ + thread: { + id: threadId, + projectId: ProjectId.make("project:executor-recovery"), + providerInstanceId: instanceId, + archivedAt: null, + deletedAt: null, + }, + runs: [ + { + id: runId, + providerInstanceId: instanceId, + status: "cancelled", + requestedAt: DateTime.subtract(now, { minutes: 22 }), + completedAt: now, + ordinal: 1, + }, + ], + messages: [], + providerTurns: [], + runtimeRequests: [], + turnItems: [], + attempts: [], + } as unknown as OrchestrationV2ThreadProjection), + dispatch: () => + Ref.update(events, (entries) => [...entries, "dispatch"]).pipe(Effect.as({} as never)), + recoverDelegatedTask: () => + Ref.update(events, (entries) => [...entries, "recover-child"]), + }, + }); + yield* Effect.gen(function* () { + const executor = yield* EffectWorker.OrchestrationEffectExecutorV2; + yield* executor.execute({ + ...restartEffect(now, { type: "detach" }), + request: { type: "provider-runtime.continue", sourceRunId: runId, processLossCount }, + }); + }).pipe(Effect.provide(layer)); + assert.deepEqual( + yield* Ref.get(events), + processLossCount > 3 ? ["recover-child"] : ["dispatch", "recover-child"], + ); + }), +); + it.effect("settles a delegated child once its restart continuation fails for good", () => Effect.gen(function* () { const timestamp = DateTime.formatIso(yield* DateTime.now); diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 42128b883711..161b2f358689 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -100,7 +100,13 @@ export const layerExecutor: Layer.Layer< switch (effect.request.type) { case "provider-runtime.continue": { const sourceRunId = effect.request.sourceRunId; - return continueRestartedRun({ threadId: effect.threadId, sourceRunId }).pipe( + return continueRestartedRun({ + threadId: effect.threadId, + sourceRunId, + ...(effect.request.processLossCount === undefined + ? {} + : { processLossCount: effect.request.processLossCount }), + }).pipe( Effect.provideService(ThreadManagementService.ThreadManagementService, threads), Effect.provideService(ServerSettings.ServerSettingsService, settings), Effect.tapError(() => diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index a37bcf8f2cd8..88fe64bf58c7 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -1858,6 +1858,142 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }), ); + it.effect("admits one restart recovery across threads until its run settles", () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const sink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:recovery-slot:a"); + const sourceRunId = RunId.make("run:recovery-slot:source"); + yield* outbox.enqueue( + [0, 1].map((index) => ({ + id: `effect:recovery-slot:${index}`, + commandId: CommandId.make(`command:recovery-slot:${index}`), + threadId: index === 0 ? threadId : ThreadId.make("thread:recovery-slot:b"), + request: { type: "provider-runtime.continue" as const, sourceRunId }, + })), + ); + const claims = yield* Effect.all( + ["a", "b"].map((workerId) => outbox.claimNext({ workerId, leaseDurationMs: 30_000 })), + { concurrency: "unbounded" }, + ); + assert.equal(claims.filter(Option.isSome).length, 1); + const claim = claims.find(Option.isSome)!; + const workerId = claim.value.leaseOwner!; + const run: OrchestrationV2Run = { + id: RunId.make("run:recovery-slot:continued"), + threadId, + ordinal: 1, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make("message:recovery-slot"), + rootNodeId: null, + activeAttemptId: null, + status: "starting", + requestedAt: now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + restartContinuationOfRunId: sourceRunId, + }; + yield* sink.write({ + events: [ + threadCreatedEvent({ + id: "event:recovery-slot:thread", + thread: makeThread(threadId, now), + now, + }), + { + id: EventId.make("event:recovery-slot:run"), + type: "run.created", + threadId, + occurredAt: now, + payload: run, + }, + ], + }); + yield* outbox.succeed({ effectId: claim.value.id, workerId }); + assert.isTrue( + Option.isNone(yield* outbox.claimNext({ workerId: "waiting", leaseDurationMs: 30_000 })), + ); + assert.isTrue(Option.isNone(yield* outbox.nextClaimableAt)); + const pending = yield* outbox.get("effect:recovery-slot:1"); + assert.isTrue(Option.isSome(pending)); + if (Option.isSome(pending)) assert.equal(pending.value.attemptCount, 0); + yield* outbox.enqueue([ + { + id: "effect:recovery-slot:manual", + commandId: CommandId.make("command:recovery-slot:manual"), + threadId: ThreadId.make("thread:recovery-slot:b"), + request: { type: "provider-turn.start", runId: RunId.make("run:recovery-slot:manual") }, + }, + ]); + const manual = yield* outbox.claimNext({ workerId: "manual", leaseDurationMs: 30_000 }); + assert.isTrue(Option.isSome(manual)); + if (Option.isSome(manual)) { + assert.equal(manual.value.id, "effect:recovery-slot:manual"); + yield* outbox.succeed({ effectId: manual.value.id, workerId: "manual" }); + } + yield* sink.write({ + events: [ + { + id: EventId.make("event:recovery-slot:done"), + type: "run.updated", + threadId, + occurredAt: now, + payload: { ...run, status: "completed", completedAt: now }, + }, + ], + }); + const next = yield* outbox.claimNext({ workerId: "next", leaseDurationMs: 30_000 }); + assert.isTrue(Option.isSome(next)); + if (Option.isSome(next)) yield* outbox.succeed({ effectId: next.value.id, workerId: "next" }); + }), + ); + + it.effect("recovers a waiting parent's child first without occupying a worker", () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const sql = yield* SqlClient.SqlClient; + const parent = ThreadId.make("thread:recovery-order:parent"); + const child = ThreadId.make("thread:recovery-order:child"); + const timestamp = DateTime.formatIso(yield* DateTime.now); + yield* sql` + INSERT INTO orchestration_v2_projection_subagents + (subagent_id, thread_id, parent_node_id, provider, child_thread_id, origin, status, updated_at, payload_json) + VALUES ('subagent:recovery-order', ${parent}, 'node:recovery-order', 'codex', ${child}, 'app_owned', 'running', ${timestamp}, '{}') + `; + yield* outbox.enqueue( + [parent, child].map((threadId, index) => ({ + id: `effect:recovery-order:${index}`, + commandId: CommandId.make(`command:recovery-order:${index}`), + threadId, + request: { + type: "provider-runtime.continue" as const, + sourceRunId: RunId.make(`run:recovery-order:${index}`), + }, + })), + ); + const first = yield* outbox.claimNext({ workerId: "child", leaseDurationMs: 30_000 }); + assert.isTrue(Option.isSome(first)); + if (Option.isNone(first)) return; + assert.equal(first.value.threadId, child); + assert.isTrue( + Option.isNone(yield* outbox.claimNext({ workerId: "parent", leaseDurationMs: 30_000 })), + ); + yield* outbox.succeed({ effectId: first.value.id, workerId: "child" }); + const next = yield* outbox.claimNext({ workerId: "parent", leaseDurationMs: 30_000 }); + assert.isTrue(Option.isSome(next)); + if (Option.isSome(next)) { + assert.equal(next.value.threadId, parent); + yield* outbox.succeed({ effectId: next.value.id, workerId: "parent" }); + } + yield* sql`DELETE FROM orchestration_v2_projection_subagents WHERE subagent_id = 'subagent:recovery-order'`; + }), + ); + it.effect("wakes a settle waiter when its effect finishes", () => Effect.gen(function* () { const outbox = yield* EffectOutbox.EffectOutboxV2; @@ -2116,6 +2252,40 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }).pipe(Effect.provide(layerIsolatedOutbox)), ); + it.effect("keeps a live continuation's crash budget past outbox retention", () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const sql = yield* SqlClient.SqlClient; + const now = yield* DateTime.now; + const old = DateTime.formatIso(DateTime.subtractDuration(now, Duration.days(8))); + const timestamp = DateTime.formatIso(now); + yield* sql` + INSERT INTO orchestration_v2_effect_outbox + (effect_id, command_id, thread_id, effect_type, payload_json, status, available_at, created_at, updated_at, completed_at) + VALUES ('effect:retained-recovery', 'command:retained-recovery', 'thread:retained-recovery', + 'provider-runtime.continue', '{"type":"provider-runtime.continue","sourceRunId":"run:retained-source","processLossCount":3}', + 'succeeded', ${old}, ${old}, ${old}, ${old}) + `; + yield* sql` + INSERT INTO orchestration_v2_projection_runs + (run_id, thread_id, ordinal, provider, status, requested_at, payload_json) + VALUES ('run:retained-recovery', 'thread:retained-recovery', 1, 'codex', 'running', ${timestamp}, + '{"restartContinuationOfRunId":"run:retained-source"}') + `; + assert.equal(yield* outbox.pruneSettled, 0); + const kept = yield* outbox.get("effect:retained-recovery"); + assert.isTrue(Option.isSome(kept)); + if (Option.isSome(kept)) + assert.deepEqual(kept.value.request, { + type: "provider-runtime.continue", + sourceRunId: RunId.make("run:retained-source"), + processLossCount: 3, + }); + yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'completed' WHERE run_id = 'run:retained-recovery'`; + assert.equal(yield* outbox.pruneSettled, 1); + }).pipe(Effect.provide(layerIsolatedOutbox)), + ); + it.effect("wakes claimers when cancellation unblocks same-thread work", () => Effect.gen(function* () { const outbox = yield* EffectOutbox.EffectOutboxV2; @@ -2473,6 +2643,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { assert.deepEqual(effects[0]?.request, { type: "provider-runtime.continue", sourceRunId: runId, + processLossCount: 0, }); yield* outbox.cancelUnsettled({ threadId, @@ -2534,7 +2705,11 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { it.effect.each([ { type: "terminal.cleanup" }, - { type: "provider-runtime.continue", sourceRunId: RunId.make("run:restart-replay") }, + { + type: "provider-runtime.continue", + sourceRunId: RunId.make("run:restart-replay"), + processLossCount: 3, + }, ] as const)( "retires live provider effects and requeues $type after process loss", (replayRequest) => diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 33b434169fed..5a5fc0cdc251 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1168,6 +1168,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? undefined : projection.messages.find((candidate) => candidate.id === delivery.messageId); + const restartCancelledRunNeedsResume = ( + projection: Pick, + source: OrchestrationV2Run, + ) => + source.status === "cancelled" && + !projection.runs.some( + (run) => + run.id !== source.id && + run.status !== "queued" && + !cancelledBeforeStart(run) && + runRanAfter(run, source), + ); + const offerDelegatedCompletionDelivery = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (threadId: ThreadId, parentRunId: RunId) { @@ -1201,6 +1214,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (providerThread === undefined) { return; } + // Recovery cancellation is not permission to start another parent turn. + // Keep the delivery durable until a continuation or a user message resumes it. + if (restartCancelledRunNeedsResume(projection, parentRun)) return; yield* continuationRequests.offer({ threadId, providerThreadId: providerThread.id, @@ -5128,6 +5144,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio generation: delivery.generation, taskIds: delivery.taskIds, }; + // A queued wake can predate recovery; enforce the hold at execution too. + if (restartCancelledRunNeedsResume(projection, parentRun!)) { + yield* emit( + events, + command, + )({ + type: "thread.metadata-updated", + threadId: command.threadId, + occurredAt: yield* DateTime.now, + payload: projection.thread, + }); + return; + } } // Route durable mailbox deliveries under the thread lock, using the live // session's capabilities. Never interrupt/restart a turn for a notification. @@ -11871,7 +11900,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); } - if (command.type === "notification.delivery.accept") { + if ( + command.type === "notification.delivery.accept" || + (command.type === "message.dispatch" && + (command.createdBy === "user" || + command.creationSource === "mcp" || + command.restartContinuationOfRunId !== undefined)) + ) { yield* mapDispatchError(command)(offerDelegatedCompletionDeliveries(command.threadId)); } if (command.type === "delegated_task.wake-policy") { diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index a3c41fed25f5..87d3f9260fb3 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -15,6 +15,7 @@ import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as EffectOutbox from "./EffectOutbox.ts"; @@ -628,13 +629,39 @@ export const make = Effect.gen(function* () { now, ) : undefined; + let processLossCount = 1; + // Normal upgrades must not exhaust an unfinished turn's crash allowance. + if ( + continuationRun?.restartContinuationOfRunId !== undefined && + continuationRun.status !== "completed" && + continuationRun.status !== "waiting" + ) { + const previous = yield* outbox + .get(`effect:restart-continuation:${continuationRun.restartContinuationOfRunId}`) + .pipe( + Effect.mapError( + (cause) => + new ProviderRuntimeRecoveryError({ + operation: "reconcile", + threadId: projection.thread.id, + cause, + }), + ), + ); + if (Option.isSome(previous) && previous.value.request.type === "provider-runtime.continue") + processLossCount += previous.value.request.processLossCount ?? 0; + } const effects: Array = continuationRun ? [ { id: `effect:restart-continuation:${continuationRun.id}`, commandId, threadId: projection.thread.id, - request: { type: "provider-runtime.continue", sourceRunId: continuationRun.id }, + request: { + type: "provider-runtime.continue", + sourceRunId: continuationRun.id, + processLossCount, + }, }, ] : []; @@ -775,7 +802,11 @@ export const make = Effect.gen(function* () { id: `effect:restart-continuation:${run.id}`, commandId, threadId, - request: { type: "provider-runtime.continue", sourceRunId: run.id }, + request: { + type: "provider-runtime.continue", + sourceRunId: run.id, + processLossCount: 0, + }, }, ], }); diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 2483534ed75d..a51a64ac998c 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -18,6 +18,7 @@ import { import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as ServerSettings from "../serverSettings.ts"; import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; @@ -288,6 +289,7 @@ it.effect.each([ assert.deepEqual(committed!.effects[0]?.request, { type: "provider-runtime.continue", sourceRunId: runId, + processLossCount: 1, }); }), ); @@ -423,7 +425,7 @@ it.effect.each(["running", "waiting"])( ]); assert.deepEqual( committed?.effects.map((effect) => effect.request), - [{ type: "provider-runtime.continue", sourceRunId: runId }], + [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 1 }], ); }), ); @@ -561,6 +563,7 @@ it.effect("prepares no continuation for background work another provider thread assert.deepEqual(writes[0]!.effects[0]?.request, { type: "provider-runtime.continue", sourceRunId: runId, + processLossCount: 0, }); }), ); @@ -666,7 +669,15 @@ it.effect.each([ const recorded = [...writes.flatMap((write) => write.effects), ...commits[0]!.effects]; assert.deepEqual( recorded.map((effect) => effect.request), - wakes ? [{ type: "provider-runtime.continue", sourceRunId: runId }] : [], + wakes + ? [ + { + type: "provider-runtime.continue", + sourceRunId: runId, + processLossCount: trigger === "shutdown" ? 0 : 1, + }, + ] + : [], ); assert.lengthOf(commits, 1); const events = commits[0]!.events; @@ -865,6 +876,76 @@ const continuationDispatches = (projection: OrchestrationV2ThreadProjection) => return dispatches; }); +it.effect.each([ + [0, "running"], + [3, "running"], + [3, "completed"], + [3, "waiting"], +] as const)("persists recovery after count %s and %s work", ([previousCount, status]) => + Effect.gen(function* () { + const base = makeProjection(); + const source = RunId.make("run:previous-loss"); + const projection = { + ...base, + runs: [ + { + ...base.runs[0]!, + status, + restartContinuationOfRunId: source, + requestedAt: DateTime.subtract(now, { minutes: 22 }), + }, + ], + providerTurns: [ + { ...base.providerTurns[0]!, status: status === "running" ? "running" : "completed" }, + ], + turnItems: status === "running" ? [] : [backgroundSubagent.turnItem], + } as unknown as OrchestrationV2ThreadProjection; + const commits: Parameters[0][] = []; + const recovery = yield* ProviderRuntimeRecovery.make.pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + commits.push(input); + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + get: (id) => { + return Effect.succeed( + id === `effect:restart-continuation:${source}` + ? Option.some({ + request: { + type: "provider-runtime.continue", + sourceRunId: source, + processLossCount: previousCount, + }, + } as EffectOutbox.OrchestrationEffectV2) + : Option.none(), + ); + }, + listByCommandId: () => Effect.succeed([]), + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + yield* recovery.reconcile("startup"); + assert.deepEqual(commits[0]?.effects[0]?.request, { + type: "provider-runtime.continue", + sourceRunId: runId, + processLossCount: status === "running" ? previousCount + 1 : 1, + }); + }), +); + const continuationTexts = (projection: OrchestrationV2ThreadProjection) => continuationDispatches(projection).pipe( Effect.map((dispatches) => dispatches.map((command) => command.text)), @@ -1046,7 +1127,7 @@ it.effect("prepares later threads' continuations when one thread fails", () => yield* recovery.prepareForShutdown; assert.deepEqual( writes.map((write) => write.effects[0]?.request), - [{ type: "provider-runtime.continue", sourceRunId: runId }], + [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 0 }], ); }), ); diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 1343fed42c58..468abd1d5b2a 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -187,7 +187,11 @@ export function restartContinuationRun( } export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")( - function* (input: { readonly threadId: ThreadId; readonly sourceRunId: RunId }) { + function* (input: { + readonly threadId: ThreadId; + readonly sourceRunId: RunId; + readonly processLossCount?: number; + }) { const settings = yield* ServerSettings.ServerSettingsService; const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null)); if (!enabled) return; @@ -240,7 +244,11 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta if (DateTime.toEpochMillis(run.requestedAt) < loopWindowStart) break; chainLength += 1; } - if (chainLength >= MAX_CONTINUATION_CHAIN) { + if ( + input.processLossCount === undefined + ? chainLength >= MAX_CONTINUATION_CHAIN + : input.processLossCount > MAX_CONTINUATION_CHAIN + ) { yield* Effect.logWarning("Not continuing a run restarted too many times in a row", { threadId: input.threadId, sourceRunId: input.sourceRunId, diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 5b8ca15241ba..dfb72631258a 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -71,6 +71,7 @@ import Migration0065 from "./Migrations/065_CodexUsageNativeTurnIndex.ts"; import Migration0066 from "./Migrations/066_RepairClientSettlementTimestamps.ts"; import Migration0067 from "./Migrations/067_WebhookDedupe.ts"; import Migration0068 from "./Migrations/068_ShellStopRequestIndex.ts"; +import Migration0069 from "./Migrations/069_RestartRecoveryIndex.ts"; const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -142,6 +143,7 @@ const migrationEntries = [ [66, "RepairClientSettlementTimestamps", Migration0066], [67, "WebhookDedupe", Migration0067], [68, "ShellStopRequestIndex", Migration0068], + [69, "RestartRecoveryIndex", Migration0069], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/069_RestartRecoveryIndex.ts b/apps/server/src/persistence/Migrations/069_RestartRecoveryIndex.ts new file mode 100644 index 000000000000..d332edde1ce2 --- /dev/null +++ b/apps/server/src/persistence/Migrations/069_RestartRecoveryIndex.ts @@ -0,0 +1,12 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + CREATE INDEX orchestration_v2_projection_runs_recovery_active_idx + ON orchestration_v2_projection_runs(status) + WHERE status IN ('starting', 'running', 'waiting') + AND json_extract(payload_json, '$.restartContinuationOfRunId') IS NOT NULL + `; +}); diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 5260ea989671..aee420269d89 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -84,14 +84,22 @@ readiness or the launcher's commit boundary. Graceful shutdown captures intent b closing providers, then reconciles after ingestion has stopped so a late completion cannot be overwritten by a stale cancellation. +The outbox reserves one recovery slot through the resulting run: limiting only dispatch +would still recreate the launch burst. Pending child recoveries precede their parent +so a parent cannot occupy the slot while waiting for a child. Explicit starts and +control effects bypass waiting recovery. + The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts) rechecks the preference, provider selection, newer user work, a stop the user requested, and maintenance turns such as `/compact` before dispatching. Intent and delivery both refuse a thread that is archived, settled, or still snoozed, a cut turn whose provider turn already ended, and a run waiting on the user: an approval, question, or secret that was pending, or that recovery closed on the cut turn. Message-capable questions stay answerable, so -they do not block. Delivery also stops after three consecutive continuations in one -`restartContinuationOfRunId` chain, so a turn that crashes the server cannot loop. Stable +they do not block. Delivery stops before the fourth consecutive process-loss recovery; +elapsed time does not reset the count. Planned shutdowns and completed turns reset it +so normal upgrades cannot exhaust an unfinished turn's allowance. A live continuation +retains its preceding intent past outbox retention to preserve that budget. Legacy +intents keep their five-minute chain guard. Stable command and message IDs prevent duplicate submissions after an outbox retry. Codex resumes without adding provider prompt text unless the turn lost background work; other adapters receive the continuation message through their normal turn path. @@ -103,3 +111,6 @@ listener ignores reconciliation's cancellations. A cancelled child whose restart continuation is still pending in the outbox is not a result yet; the continuation's run settles it, or the handler settles it when it declines to continue. Schedulers wait for activation so they cannot start runs that reconciliation would then cancel. +An unrecovered cancelled parent keeps those completion deliveries durable without +offering a wake. A continuation or a user message re-offers them; dispatch also checks +the hold because a wake can already have been queued before reconciliation. diff --git a/docs/user/updating.md b/docs/user/updating.md index 706d8fad5e10..f46923dbb47a 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -53,8 +53,16 @@ A thread that finished its turn to wait on a monitor, subagent, or background ta is woken and told that this work stopped. A background shell, such as a dev server, does not wake it; the thread learns about it with your next message. A thread is not resumed while it waits for your approval, answer, or secret, or -while it is archived, snoozed, or settled. After three automatic resumes in a row, -the thread waits for your next message. +while it is archived, snoozed, or settled. Automatic recovery resumes one thread +at a time, with pending child recoveries before their parent. After three +consecutive recoveries of an unfinished turn following crashes, it waits for your +next message, even when the restarts are far apart. A completed turn or a planned +shutdown resets that limit. + +If a parent was cancelled by a restart, child results stay saved until the parent +resumes. Turning this setting off or reaching the crash limit therefore does not +wake the parent through a cancelled child's notification. Send a message to resume +the parent manually, or use Stop to dismiss its pending child notifications. Updates from the previous orchestration system preserve conversation transcripts but cannot carry every kind of runtime history forward. Read [Threads from older T3 Code versions](./thread-migration.md)