diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 342f6ced28bae..a415b9e3420cf 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -6402,6 +6402,59 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect.each([ + { cooldown: " (reset after 1h2m3s)", seconds: 3723 }, + { cooldown: " (reset after 4m 6s) (last error: rate limited)", seconds: 246 }, + { cooldown: "", seconds: 0 }, + ])("keeps a gateway's reset on a direct 429 with cooldown '$cooldown'", (input) => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-direct-gateway-limit"), + text: "Continue.", + attachments: [], + }), + ); + // An API-key session reports a gateway 429 only as text, with no rate_limit_info. + const result = `API Error: Request rejected (429) ยท All credentials for model claude-opus-5-5 are cooling down${input.cooldown}`; + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000654", + error: "rate_limit", + text: result, + }), + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000655", + result, + terminalReason: "api_error", + isError: true, + apiErrorStatus: 429, + }), + ]); + const terminal = yield* Queue.take(harness.terminalReceipts); + assert.equal(terminal.status, "failed"); + if (terminal.status !== "failed") return; + assert.equal(terminal.failure.class, "usage_limit"); + assert.equal( + terminal.failure.resetAt, + input.seconds > 0 + ? DateTime.formatIso( + DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000), + ) + : null, + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect("keeps the measured context when a usage limit ends the turn", () => Effect.gen(function* () { const harness = yield* makeWakeHarness; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index d8a88b7dd2b96..a99f336487aab 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -3753,37 +3753,46 @@ function claudeUsageLimitResetAt(resetsAt: unknown): string | null { : null; } +function gatewayCooldownResetAt( + cooldown: RegExpExecArray | null, + completedAt: DateTime.Utc, +): string | undefined { + if (cooldown === null) return undefined; + const seconds = + Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0); + const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000; + return seconds > 0 && resetMs < 8.64e15 + ? DateTime.formatIso(DateTime.makeUnsafe(resetMs)) + : undefined; +} + function providerFailureFromResult( message: SDKResultMessage, completedAt: DateTime.Utc, failureHint?: string, usageLimited = false, ): OrchestrationV2ProviderFailure | null { - const gatewayLimit = + const gatewayResetAt = failureHint === undefined && message.subtype === "success" && message.is_error && message.terminal_reason === "api_error" && message.api_error_status === 503 - ? /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( - message.result, + ? gatewayCooldownResetAt( + /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( + message.result, + ), + completedAt, ) - : null; - if (gatewayLimit) { - const seconds = - Number(gatewayLimit[1] ?? 0) * 3_600 + - Number(gatewayLimit[2] ?? 0) * 60 + - Number(gatewayLimit[3] ?? 0); - const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000; - if (seconds > 0 && resetMs < 8.64e15) { - return makeProviderFailure({ - class: "usage_limit", - message: "Claude API rate limit reached. Try again later.", - code: "api_error_429", - retryable: true, - resetAt: DateTime.formatIso(DateTime.makeUnsafe(resetMs)), - }); - } + : undefined; + if (gatewayResetAt !== undefined) { + return makeProviderFailure({ + class: "usage_limit", + message: "Claude API rate limit reached. Try again later.", + code: "api_error_429", + retryable: true, + resetAt: gatewayResetAt, + }); } const failureClass = (message.subtype === "success" && message.api_error_status === 429) || usageLimited @@ -3812,6 +3821,14 @@ function providerFailureFromResult( return null; } const apiErrorStatus = message.api_error_status ?? null; + // API-key sessions get no rate_limit_info, so a gateway's 429 carries its reset only as text. + const resetAt = + apiErrorStatus === 429 && message.result.startsWith("API Error:") + ? gatewayCooldownResetAt( + /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result), + completedAt, + ) + : undefined; return makeProviderFailure({ message: listedError ?? structuredError ?? failureHint ?? message.result, code: @@ -3820,6 +3837,7 @@ function providerFailureFromResult( : `api_error_${apiErrorStatus}`, class: failureClass, retryable: apiErrorStatus === 429 || apiErrorStatus === 529 ? true : null, + resetAt: resetAt ?? null, }); }