Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6402,6 +6402,59 @@ describe("ClaudeAdapterV2 background wake turns", () => {
),
);

it.effect.each([
{ cooldown: " (reset after 1h2m3s)", seconds: 3723 },
{ cooldown: " (reset after 4m 6s) (last error: rate limited)", seconds: 246 },
{ cooldown: "", seconds: 0 },
])("keeps a gateway's reset on a direct 429 with cooldown '$cooldown'", (input) =>
Effect.gen(function* () {
const harness = yield* makeWakeHarness;
const now = yield* DateTime.now;
yield* harness.runtime.startTurn(
makeClaudeTestTurnInput({
threadId: harness.threadId,
providerThread: harness.providerThread,
now,
attemptId: RunAttemptId.make("attempt-direct-gateway-limit"),
text: "Continue.",
attachments: [],
}),
);
// An API-key session reports a gateway 429 only as text, with no rate_limit_info.
const result = `API Error: Request rejected (429) · All credentials for model claude-opus-5-5 are cooling down${input.cooldown}`;
yield* Queue.offerAll(harness.sdkMessages, [
makeAssistantErrorFrame({
uuid: "00000000-0000-4000-8000-000000000654",
error: "rate_limit",
text: result,
}),
makeResultFrame({
uuid: "00000000-0000-4000-8000-000000000655",
result,
terminalReason: "api_error",
isError: true,
apiErrorStatus: 429,
}),
]);
const terminal = yield* Queue.take(harness.terminalReceipts);
assert.equal(terminal.status, "failed");
if (terminal.status !== "failed") return;
assert.equal(terminal.failure.class, "usage_limit");
assert.equal(
terminal.failure.resetAt,
input.seconds > 0
? DateTime.formatIso(
DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000),
)
: null,
);
}).pipe(
Effect.provide(
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
),
),
);

it.effect("keeps the measured context when a usage limit ends the turn", () =>
Effect.gen(function* () {
const harness = yield* makeWakeHarness;
Expand Down
56 changes: 37 additions & 19 deletions apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3753,37 +3753,46 @@ function claudeUsageLimitResetAt(resetsAt: unknown): string | null {
: null;
}

function gatewayCooldownResetAt(
cooldown: RegExpExecArray | null,
completedAt: DateTime.Utc,
): string | undefined {
if (cooldown === null) return undefined;
const seconds =
Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0);
const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000;
return seconds > 0 && resetMs < 8.64e15
? DateTime.formatIso(DateTime.makeUnsafe(resetMs))
: undefined;
}

function providerFailureFromResult(
message: SDKResultMessage,
completedAt: DateTime.Utc,
failureHint?: string,
usageLimited = false,
): OrchestrationV2ProviderFailure | null {
const gatewayLimit =
const gatewayResetAt =
failureHint === undefined &&
message.subtype === "success" &&
message.is_error &&
message.terminal_reason === "api_error" &&
message.api_error_status === 503
? /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(
message.result,
? gatewayCooldownResetAt(
/^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(
message.result,
),
completedAt,
)
: null;
if (gatewayLimit) {
const seconds =
Number(gatewayLimit[1] ?? 0) * 3_600 +
Number(gatewayLimit[2] ?? 0) * 60 +
Number(gatewayLimit[3] ?? 0);
const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000;
if (seconds > 0 && resetMs < 8.64e15) {
return makeProviderFailure({
class: "usage_limit",
message: "Claude API rate limit reached. Try again later.",
code: "api_error_429",
retryable: true,
resetAt: DateTime.formatIso(DateTime.makeUnsafe(resetMs)),
});
}
: undefined;
if (gatewayResetAt !== undefined) {
return makeProviderFailure({
class: "usage_limit",
message: "Claude API rate limit reached. Try again later.",
code: "api_error_429",
retryable: true,
resetAt: gatewayResetAt,
});
}
const failureClass =
(message.subtype === "success" && message.api_error_status === 429) || usageLimited
Expand Down Expand Up @@ -3812,6 +3821,14 @@ function providerFailureFromResult(
return null;
}
const apiErrorStatus = message.api_error_status ?? null;
// API-key sessions get no rate_limit_info, so a gateway's 429 carries its reset only as text.
const resetAt =
apiErrorStatus === 429 && message.result.startsWith("API Error:")
? gatewayCooldownResetAt(
/\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result),
completedAt,
)
: undefined;
return makeProviderFailure({
message: listedError ?? structuredError ?? failureHint ?? message.result,
code:
Expand All @@ -3820,6 +3837,7 @@ function providerFailureFromResult(
: `api_error_${apiErrorStatus}`,
class: failureClass,
retryable: apiErrorStatus === 429 || apiErrorStatus === 529 ? true : null,
resetAt: resetAt ?? null,
});
}

Expand Down
Loading