diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 7fece9817ce01..e0e4b268f6998 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -5526,6 +5526,60 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect.each([ + { suffix: " (reset after 2h52m31s)", seconds: 10351 }, + { suffix: " (reset after 72h0m0s)", seconds: 259200 }, + { suffix: " (reset after 0s)", seconds: 0 }, + { suffix: "", seconds: 0 }, + ])("reads a gateway cooldown from a 429 result$suffix", (input) => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-gateway-429"), + text: "Continue.", + attachments: [], + }), + ); + // CLIProxyAPI's answer once every pooled account is cooling down. + const text = `API Error: Request rejected (429) ยท All credentials for model claude-opus-5-5 are cooling down (last error: rate_limit_error: This request would exceed your account's rate limit. Please try again later.)${input.suffix}`; + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000654", + error: "rate_limit", + text, + }), + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000655", + result: text, + terminalReason: "api_error", + isError: true, + apiErrorStatus: 429, + }), + ]); + const terminal = yield* Queue.take(harness.terminalReceipts); + assert.equal(terminal.status, "failed"); + if (terminal.status !== "failed") return; + assert.equal(terminal.failure.class, "usage_limit"); + assert.equal( + terminal.failure.resetAt, + input.seconds > 0 + ? DateTime.formatIso( + DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000), + ) + : null, + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect("keeps the measured context when a usage limit ends the turn", () => Effect.gen(function* () { const harness = yield* makeWakeHarness; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index baf9a0b0a690b..02c2873c3bb41 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -3516,37 +3516,43 @@ function claudeUsageLimitResetAt(resetsAt: unknown): string | null { : null; } +// Gateways that pool Claude accounts (9router, CLIProxyAPI) answer a limit with their own +// cooldown in the error text, e.g. "(reset after 1h 2m 3s)", since the CLI drops Retry-After. +function gatewayCooldownResetAt(text: string, completedAt: DateTime.Utc): string | null { + const cooldown = /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(text); + if (!cooldown) return null; + const seconds = + Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0); + const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000; + return seconds > 0 && resetMs < 8.64e15 ? DateTime.formatIso(DateTime.makeUnsafe(resetMs)) : null; +} + function providerFailureFromResult( message: SDKResultMessage, completedAt: DateTime.Utc, failureHint?: string, usageLimited = false, ): OrchestrationV2ProviderFailure | null { - const gatewayLimit = + // 9router wraps the upstream 429 in a 503. + const wrappedGatewayResetAt = failureHint === undefined && message.subtype === "success" && message.is_error && message.terminal_reason === "api_error" && - message.api_error_status === 503 - ? /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( - message.result, - ) + message.api_error_status === 503 && + /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after /.test( + message.result, + ) + ? gatewayCooldownResetAt(message.result, completedAt) : null; - if (gatewayLimit) { - const seconds = - Number(gatewayLimit[1] ?? 0) * 3_600 + - Number(gatewayLimit[2] ?? 0) * 60 + - Number(gatewayLimit[3] ?? 0); - const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000; - if (seconds > 0 && resetMs < 8.64e15) { - return makeProviderFailure({ - class: "usage_limit", - message: "Claude API rate limit reached. Try again later.", - code: "api_error_429", - retryable: true, - resetAt: DateTime.formatIso(DateTime.makeUnsafe(resetMs)), - }); - } + if (wrappedGatewayResetAt !== null) { + return makeProviderFailure({ + class: "usage_limit", + message: "Claude API rate limit reached. Try again later.", + code: "api_error_429", + retryable: true, + resetAt: wrappedGatewayResetAt, + }); } const failureClass = (message.subtype === "success" && message.api_error_status === 429) || usageLimited @@ -3583,6 +3589,7 @@ function providerFailureFromResult( : `api_error_${apiErrorStatus}`, class: failureClass, retryable: apiErrorStatus === 429 || apiErrorStatus === 529 ? true : null, + resetAt: apiErrorStatus === 429 ? gatewayCooldownResetAt(message.result, completedAt) : null, }); }