From 52a9328ebf7d8c0ab82f8edd154956221c6ac141 Mon Sep 17 00:00:00 2001 From: Artur Safonov Date: Sun, 25 Oct 2026 14:26:48 +0200 Subject: [PATCH 1/5] feat(combos): add opt-in interval-limited cooldown reprobe --- docs-site/src/content/docs/guides/combos.md | 28 +++++ scripts/test-layout/layout.json | 1 + src/cli/capabilities-agents-routing.ts | 4 +- src/cli/combo-result.ts | 7 ++ src/cli/combo.ts | 18 +++ src/combos/resolve.ts | 29 +++++ src/combos/types.ts | 24 ++++ src/server/management/combo-routes.ts | 4 + src/types/config.ts | 6 + structure/fallback-policy.md | 2 + structure/providers-and-adapters.md | 2 +- structure/runtime.md | 2 +- tests/cli/cli-combo-parity.test.ts | 32 +++++ .../combo-cooldown-reprobe.test.ts | 116 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + 15 files changed, 272 insertions(+), 4 deletions(-) create mode 100644 tests/codex-integration/combo-cooldown-reprobe.test.ts diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 55faba32e8d..de670cbd17c 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -1029,3 +1029,31 @@ validation message. The error was terminal rather than target-specific. Fix invalid input, reduce an oversized context, handle a policy refusal, or correct the rejected request origin. Combos do not hop for those cases. + +## Retry all-cooled targets (opt-in) + +Combo cooldown recovery is opt-in and disabled by default. To allow one new +failover pass when every otherwise eligible target has a saved combo cooldown: + +```sh +ocx combo set review --cooldown-reprobe on --cooldown-reprobe-interval 300000 +ocx combo set review --cooldown-reprobe off +``` + +The corresponding per-combo configuration is: + +```json +{ "cooldownReprobe": { "enabled": true, "intervalMs": 300000 } } +``` + +The default interval is five minutes; accepted values are integer milliseconds +from 1000 to 86400000. Setting the interval alone does not enable recovery. +Recovery is demand-driven: the first eligible request can reserve a pass immediately, +and later passes are separated by at least the configured interval. There is no timer +or background traffic. Each target can be attempted once in the request's existing +failover budget; failures record fresh cooldowns. Account quota checks, disabled +providers, request compatibility, metered-spend permissions and cancellation remain +enforced. Explicit provider Retry-After values may therefore be retried early only +when the operator enables this option. Preview/list operations never clear cooldowns. +The interval is tracked per combo in memory and resets on proxy restart or combo +selection-state reset. Existing cooldown-wait behavior is unchanged when disabled. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 2fdded5a374..b200958a8c6 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -741,6 +741,7 @@ "cold-spawn-warmup.test.ts": "ci-workflows", "combo-authoritative-reset.test.ts": "codex-integration", "combo-codex-exhaustion-cooldown.test.ts": "codex-integration", + "combo-cooldown-reprobe.test.ts": "codex-integration", "combo-permanent-failure-cooldown.test.ts": "codex-integration", "combo-reset-window-model-scope.test.ts": "codex-integration", "combo-child-headers.test.ts": "routing", diff --git a/src/cli/capabilities-agents-routing.ts b/src/cli/capabilities-agents-routing.ts index ff5adad3fa4..a49ba0a3470 100644 --- a/src/cli/capabilities-agents-routing.ts +++ b/src/cli/capabilities-agents-routing.ts @@ -385,10 +385,10 @@ export const AGENT_ROUTING_CAPABILITIES: readonly Capability[] = [ }, { command: ["combo","set"], - usage: "ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] [--effort ] [--effort-mode ] [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] [--description ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] [--session-pin ] [--rename-from ] [--image-input ] [--reasoning-effort-mode ] [--json]", + usage: "ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] [--cooldown-reprobe ] [--cooldown-reprobe-interval ] [--effort ] [--effort-mode ] [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] [--description ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] [--session-pin ] [--rename-from ] [--image-input ] [--reasoning-effort-mode ] [--json]", summary: "Create, update or rename a combo.", routes: [{"method":"GET","path":"/api/combos"},{"method":"PUT","path":"/api/combos"}], - flags: [{"name":"--allow-metered-targets","value":"string","summary":"on grants operator consent to metered targets only on a loopback-only proxy; off clears it."},{"name":"--targets","value":"string","summary":"Ordered provider/model[:weight] list; required for creation."},{"name":"--strategy","value":"string","summary":"Routing strategy."},{"name":"--sticky","value":"string","summary":"Round-robin limit 1-100; - resets."},{"name":"--effort","value":"string","summary":"Default effort; - clears."},{"name":"--effort-mode","value":"string","summary":"fallback or force; - resets to fallback."},{"name":"--alias","value":"string","summary":"Alias; - clears."},{"name":"--display-name","value":"string","summary":"Label; - clears."},{"name":"--description","value":"string","summary":"Catalog row description; - clears."},{"name":"--decision-provider","value":"string","summary":"JEV provider; - clears."},{"name":"--decision-model","value":"string","summary":"JEV route; - clears; exclusive with provider."},{"name":"--decision-timeout","value":"string","summary":"JEV deadline 1000-120000 ms; - clears."},{"name":"--decision-quota-signals","value":"string","summary":"JEV advisory quota on/off; - clears."},{"name":"--decision-quota-tiers","value":"string","summary":"Partial JSON thresholds; - clears."},{"name":"--session-pin","value":"string","summary":"JEV session pin idle window 1m-24h (90s, 20m, 1h); - clears."},{"name":"--rename-from","value":"string","summary":"Existing combo ID to rename."},{"name":"--json","value":"boolean","summary":"Emit structured JSON."},{"name":"--native-alias","value":"string","summary":"Bare legacy flag means on; explicit on/off preserves presence, including false."},{"name":"--targets-file","value":"string","summary":"Bounded ordered target JSON array, exclusive with --targets; preserves effort/profile/last-resort metadata."},{"name":"--image-input","value":"string","summary":"auto or disabled; explicit auto clears the previous disabled choice."},{"name":"--reasoning-effort-mode","value":"string","summary":"strict or adaptive, distinct from --effort-mode fallback/force."}], + flags: [{"name":"--allow-metered-targets","value":"string","summary":"on grants operator consent to metered targets only on a loopback-only proxy; off clears it."},{"name":"--targets","value":"string","summary":"Ordered provider/model[:weight] list; required for creation."},{"name":"--strategy","value":"string","summary":"Routing strategy."},{"name":"--sticky","value":"string","summary":"Round-robin limit 1-100; - resets."},{"name":"--cooldown-reprobe","value":"string","summary":"Opt into one recovery pass when all eligible targets cool; off by default."},{"name":"--cooldown-reprobe-interval","value":"string","summary":"Minimum interval between recovery passes in ms (1000..86400000); default 300000."},{"name":"--effort","value":"string","summary":"Default effort; - clears."},{"name":"--effort-mode","value":"string","summary":"fallback or force; - resets to fallback."},{"name":"--alias","value":"string","summary":"Alias; - clears."},{"name":"--display-name","value":"string","summary":"Label; - clears."},{"name":"--description","value":"string","summary":"Catalog row description; - clears."},{"name":"--decision-provider","value":"string","summary":"JEV provider; - clears."},{"name":"--decision-model","value":"string","summary":"JEV route; - clears; exclusive with provider."},{"name":"--decision-timeout","value":"string","summary":"JEV deadline 1000-120000 ms; - clears."},{"name":"--decision-quota-signals","value":"string","summary":"JEV advisory quota on/off; - clears."},{"name":"--decision-quota-tiers","value":"string","summary":"Partial JSON thresholds; - clears."},{"name":"--session-pin","value":"string","summary":"JEV session pin idle window 1m-24h (90s, 20m, 1h); - clears."},{"name":"--rename-from","value":"string","summary":"Existing combo ID to rename."},{"name":"--json","value":"boolean","summary":"Emit structured JSON."},{"name":"--native-alias","value":"string","summary":"Bare legacy flag means on; explicit on/off preserves presence, including false."},{"name":"--targets-file","value":"string","summary":"Bounded ordered target JSON array, exclusive with --targets; preserves effort/profile/last-resort metadata."},{"name":"--image-input","value":"string","summary":"auto or disabled; explicit auto clears the previous disabled choice."},{"name":"--reasoning-effort-mode","value":"string","summary":"strict or adaptive, distinct from --effort-mode fallback/force."}], mutates: true, json: "payload", details: ["create and update are aliases of set. Existing fields are read before writing; --targets replaces targets. Without --strategy an existing combo keeps its stored strategy and JEV policy; a new combo defaults to failover.","force overrides valid client effort and may increase cost and latency.","Only the implemented target grammar is supported; full GUI target metadata and routing-profile writes are separate tasks.","New file/flag conflicts refuse before transport. Partial edits preserve target metadata; explicit false/default values override server carry. Turning a native alias off may also require clearing its incompatible alias.","Read/merge/write uses one target and remains an upsert without revision CAS. Saved-but-incomplete catalog outcomes return nonzero with their receipt."], diff --git a/src/cli/combo-result.ts b/src/cli/combo-result.ts index b98b9560547..e077cd041d1 100644 --- a/src/cli/combo-result.ts +++ b/src/cli/combo-result.ts @@ -40,6 +40,13 @@ function publicCombo(value: unknown): Record { if (!tiers) invalid(); result[key] = tiers; continue; + } else if (key === "cooldownReprobe") { + if (!record(child) || typeof child.enabled !== "boolean" + || Object.keys(child).some(field => field !== "enabled" && field !== "intervalMs")) invalid(); + if (child.intervalMs !== undefined && (typeof child.intervalMs !== "number" + || !Number.isSafeInteger(child.intervalMs) || child.intervalMs < 1000 || child.intervalMs > 86_400_000)) invalid(); + result[key] = { enabled: child.enabled, ...(child.intervalMs !== undefined ? { intervalMs: child.intervalMs } : {}) }; + continue; } else if (key === "sessionPin") { const pin = normalizeJevSessionPin(child); if (!pin) invalid(); diff --git a/src/cli/combo.ts b/src/cli/combo.ts index 84812e86d33..a97174942b7 100644 --- a/src/cli/combo.ts +++ b/src/cli/combo.ts @@ -24,6 +24,7 @@ const USAGE = `Usage: ocx combo show [--json] ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] + [--cooldown-reprobe ] [--cooldown-reprobe-interval ] [--effort ] [--effort-mode ] (force overrides valid client effort and can increase cost/latency) [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] @@ -114,6 +115,16 @@ async function set(argv: string[], deps: RuntimeApiDeps): Promise { throw new CliUsageError("--sticky must be an integer from 1 to 100, or -", USAGE); } if (stickyLimit !== undefined && stickyLimit > 100) throw new CliUsageError("--sticky must be <= 100", USAGE); + const reprobe = takeOption(args, "--cooldown-reprobe"); + const reprobeInterval = takeOption(args, "--cooldown-reprobe-interval"); + if (reprobe !== undefined && reprobe !== "on" && reprobe !== "off") { + throw new CliUsageError("--cooldown-reprobe must be on or off", USAGE); + } + const reprobeIntervalMs = reprobeInterval === undefined ? undefined : Number(reprobeInterval); + if (reprobeIntervalMs !== undefined && (!Number.isSafeInteger(reprobeIntervalMs) + || reprobeIntervalMs < 1000 || reprobeIntervalMs > 86_400_000)) { + throw new CliUsageError("--cooldown-reprobe-interval must be an integer from 1000 to 86400000 ms", USAGE); + } const effort = takeOption(args, "--effort"); const effortMode = takeOption(args, "--effort-mode"); if (effortMode !== undefined && effortMode !== "fallback" && effortMode !== "force" && effortMode !== "-") { @@ -180,6 +191,13 @@ async function set(argv: string[], deps: RuntimeApiDeps): Promise { const combo: Record = existing ? Object.fromEntries(Object.entries(existing).filter(([, value]) => value !== null)) : { strategy, stickyLimit: stickyLimit ?? 1 }; + if (reprobe !== undefined || reprobeIntervalMs !== undefined) { + const previous = (combo.cooldownReprobe ?? {}) as { enabled?: boolean; intervalMs?: number }; + combo.cooldownReprobe = { ...previous, + enabled: reprobe === undefined ? previous.enabled === true : reprobe === "on", + ...(reprobeIntervalMs !== undefined ? { intervalMs: reprobeIntervalMs } : {}), + }; + } delete combo.id; delete combo.model; if (replacingTargets) combo.targets = additional.targets ?? parsedTargets; diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index bf7bfe125f6..c8cd848bf5c 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -3,6 +3,7 @@ import { getCachedProviderRoutingQuota } from "../providers/quota-routing-cache" import type { ProviderQuota, ProviderQuotaWindow } from "../providers/quota-types"; import { sleepWithAbort } from "../lib/upstream-retry"; import { + clearComboTargetCooldowns, coolComboTarget, earliestComboCooldown, isComboTargetInCooldown, @@ -26,6 +27,7 @@ export interface ComboPick { } interface SelectionState { + lastCooldownReprobeAt?: number; activeKey?: string; successes: number; currentWeights: Map; @@ -286,6 +288,27 @@ function resetWindowIndex( return selected; } +/** Reserve one recovery pass before mutating cooldowns; preview and in-flight picks never call this. */ +function reopenAllCooledTargets( + config: OcxConfig, comboId: string, now: number, + eligible?: (target: NormalizedComboTarget) => boolean, +): void { + const combo = getCombo(config, comboId); + if (!combo?.cooldownReprobe?.enabled) return; + const state = selectionState.get(comboId); + if (state?.lastCooldownReprobeAt !== undefined + && now - state.lastCooldownReprobeAt < combo.cooldownReprobe.intervalMs) return; + const usable = combo.targets.filter(target => targetProviderIsUsable(config, target, now) + && (eligible?.(target) ?? true)); + if (usable.length === 0 || !usable.every(target => isComboTargetInCooldown(comboId, target, now))) return; + // No await between reservation and reset: concurrent requests cannot reserve another pass. + const next = state ?? { successes: 0, currentWeights: new Map(), successfulUses: new Map() }; + next.lastCooldownReprobeAt = now; + selectionState.set(comboId, next); + clearComboTargetCooldowns(comboId); + console.warn(`[combo] ${comboId}: reopening all-cooled targets; minimum reprobe interval ${combo.cooldownReprobe.intervalMs}ms`); +} + export function pickComboTarget( config: OcxConfig, comboId: string, @@ -302,6 +325,9 @@ export function pickComboTarget( if (!combo) throw new UnknownComboError(comboId); const excluded = new Set(options.exclude ?? []); const now = options.now ?? Date.now(); + if (!options.preview && excluded.size === 0) { + reopenAllCooledTargets(config, comboId, now, options.eligible); + } const eligible = (target: NormalizedComboTarget): boolean => targetProviderIsUsable(config, target, now) && !isComboTargetInCooldown(comboId, target, now) @@ -505,6 +531,9 @@ export async function pickComboTargetWithWait( const now = options.now ?? Date.now(); const excluded = new Set(options.exclude ?? []); const customEligible = options.eligible; + if (excluded.size === 0 && !options.abortSignal?.aborted) { + reopenAllCooledTargets(config, comboId, now, customEligible); + } const eligibleAt = (target: NormalizedComboTarget, at: number): boolean => !isComboTargetInCooldown(comboId, target, at) && (customEligible?.(target) ?? true); diff --git a/src/combos/types.ts b/src/combos/types.ts index f3256fe43c8..7f35f915a18 100644 --- a/src/combos/types.ts +++ b/src/combos/types.ts @@ -17,6 +17,7 @@ import { } from "./jev-decision-contract"; export const COMBO_DEFAULT_WAIT_FOR_COOLDOWN_MS = 0; +export const COMBO_DEFAULT_COOLDOWN_REPROBE_INTERVAL_MS = 300_000; export const JEV_MAX_CANDIDATE_FIELD_CHARS = 512; export { JEV_DECISION_TIMEOUT_DEFAULT_MS, @@ -60,6 +61,7 @@ export interface NormalizedComboConfig { stickyLimit: number; cooldownMs?: number; waitForCooldownMs: number; + cooldownReprobe?: { enabled: true; intervalMs: number }; /** `before-last-resort` defers lastResort targets while a normal one can be waited out (#5691). */ cooldownWaitPolicy: OcxComboCooldownWaitPolicy | null; defaultEffort: OcxComboDefaultEffort | null; @@ -222,6 +224,24 @@ export function comboConfigIssues( || body.waitForCooldownMs > 600_000)) { issues.push({ path: ["waitForCooldownMs"], message: "waitForCooldownMs must be an integer from 0 to 600000" }); } + if (body.cooldownReprobe !== undefined) { + const value = body.cooldownReprobe; + if (!value || typeof value !== "object" || Array.isArray(value)) { + issues.push({ path: ["cooldownReprobe"], message: "cooldownReprobe must be an object" }); + } else { + const reprobe = value as Record; + if (typeof reprobe.enabled !== "boolean") { + issues.push({ path: ["cooldownReprobe", "enabled"], message: "cooldownReprobe.enabled must be a boolean" }); + } + if (reprobe.intervalMs !== undefined && (typeof reprobe.intervalMs !== "number" + || !Number.isSafeInteger(reprobe.intervalMs) || reprobe.intervalMs < 1000 || reprobe.intervalMs > 86_400_000)) { + issues.push({ path: ["cooldownReprobe", "intervalMs"], message: "cooldownReprobe.intervalMs must be an integer from 1000 to 86400000" }); + } + if (Object.keys(reprobe).some(key => key !== "enabled" && key !== "intervalMs")) { + issues.push({ path: ["cooldownReprobe"], message: "cooldownReprobe supports only enabled and intervalMs" }); + } + } + } if (body.cooldownWaitPolicy !== undefined && body.cooldownWaitPolicy !== null && body.cooldownWaitPolicy !== "before-last-resort") { issues.push({ @@ -535,6 +555,10 @@ export function normalizeComboConfig(raw: OcxComboConfig): NormalizedComboConfig ...(raw.allowMeteredTargets === true ? { allowMeteredTargets: true } : {}), stickyLimit: raw.stickyLimit ?? 1, cooldownMs: raw.cooldownMs, + ...(raw.cooldownReprobe?.enabled === true ? { cooldownReprobe: { + enabled: true as const, + intervalMs: raw.cooldownReprobe.intervalMs ?? COMBO_DEFAULT_COOLDOWN_REPROBE_INTERVAL_MS, + } } : {}), waitForCooldownMs: raw.waitForCooldownMs ?? COMBO_DEFAULT_WAIT_FOR_COOLDOWN_MS, cooldownWaitPolicy: raw.cooldownWaitPolicy === "before-last-resort" ? "before-last-resort" : null, defaultEffort, diff --git a/src/server/management/combo-routes.ts b/src/server/management/combo-routes.ts index bcb1de6da21..9336019b4e8 100644 --- a/src/server/management/combo-routes.ts +++ b/src/server/management/combo-routes.ts @@ -222,6 +222,10 @@ export async function handleComboRoutes(ctx: ManagementContext): Promise = body.combo; const effectiveCombo = { ...requestedCombo, + // Clients that do not expose this opt-in must preserve it on unrelated edits. + ...(!Object.hasOwn(requestedCombo, "cooldownReprobe") && previous?.cooldownReprobe !== undefined + ? { cooldownReprobe: previous.cooldownReprobe } + : {}), ...(!Object.hasOwn(requestedCombo, "decisionQuotaSignals") && requestedCombo.strategy === "jev" && previous?.decisionQuotaSignals !== undefined ? { decisionQuotaSignals: previous.decisionQuotaSignals } : {}), ...(!Object.hasOwn(requestedCombo, "decisionQuotaTiers") && requestedCombo.strategy === "jev" && previous?.decisionQuotaTiers !== undefined ? { decisionQuotaTiers: previous.decisionQuotaTiers } : {}), // The dashboard does not edit the session pin (#7525): an omitted field keeps it on a JEV save. diff --git a/src/types/config.ts b/src/types/config.ts index 479895aa2c4..eb1f917534f 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1435,6 +1435,12 @@ export interface OcxComboConfig { cooldownMs?: number; /** Maximum wait for an eligible target cooldown to expire before failing closed. Default 0; range 0..600000, per selection attempt. */ waitForCooldownMs?: number; + /** Opt-in retry pass when all otherwise eligible targets are cooling. Disabled by default. */ + cooldownReprobe?: { + enabled: boolean; + /** Minimum interval between recovery passes, in milliseconds. Default 300000; range 1000..86400000. */ + intervalMs?: number; + }; /** * `before-last-resort` defers targets marked `lastResort` while a normal * target is merely cooling and that cooldown can be waited out inside diff --git a/structure/fallback-policy.md b/structure/fallback-policy.md index fc7ec6f9c38..ae36c897249 100644 --- a/structure/fallback-policy.md +++ b/structure/fallback-policy.md @@ -129,3 +129,5 @@ privacy), `tests/responses/responses-combo-fallback-off.test.ts`, `tests/routing `tests/routing/subagent-model-fallback-off.test.ts`, `tests/codex-integration/main-account-hard-lock-fallback-off.test.ts`, `tests/server/fallback-routes.test.ts`, `tests/clients/client-home-unreachable.test.ts`, `tests/cli/cli-fallback.test.ts` and `gui/tests/fallback-panel.test.tsx`. + +The opt-in [all-cooled Combo recovery](runtime.md#scoped-provider-quota-for-combo-selection) can reopen saved combo cooldowns in `src/combos/resolve.ts`; it does not authorize a fallback hop or bypass the configured fallback policy. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 8dae56c743c..3ebcbeb8246 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -589,7 +589,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr remains authoritative. No raw body is retained in that field, serialized into client errors, or logged. Presence of both `detail` and `error` across the original root and its own non-array `response` record is ambiguous, regardless of values; null counts and deeper quoted objects do not. This is checked before selecting an error carrier. For complete bounded HTTP errors, the text wrapper selects root `detail`/`error` when present; otherwise it selects the own `response` record for the same exact `detail` or `error.message` match. It never searches another carrier after a nonmatch or descends further. Policy-fallback and direct callers share this wrapper; SSE message parsing disables nested HTTP selection. The same bounded HTTP parser retains allowlisted dedicated `code` fields from root and response before display truncation; diagnostic `type` is never promoted, including when code is absent or malformed. Policy/direct classification inspects full text only without authoritative carrier metadata; SSE never borrows hard codes from quoted message JSON. -Committed output, cancellation, hard policy refusals and resend limits remain stronger than model fallback. +Committed output, cancellation, hard policy refusals and resend limits remain stronger than model fallback. Opt-in all-cooled target recovery in `src/combos/resolve.ts` follows [the Combo cooldown contract](runtime.md#scoped-provider-quota-for-combo-selection) without widening account/provider admission. Token Plan dated-model policy (2026-10-20): both curated presets include exact `deepseek-v4.1-flash-1017` metadata without changing live discovery or Responses pins. diff --git a/structure/runtime.md b/structure/runtime.md index 15baf9348fd..feaae862186 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. Every combo, routing-profile, subagent and hard-lock hop to another model or provider first passes the [fallback policy](fallback-policy.md#sites). -`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. +`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. Opt-in `cooldownReprobe: { enabled: true, intervalMs }` in `src/combos/resolve.ts` permits an initial non-preview request to clear all-cooled eligible targets once per configured interval (default 300000 ms, range 1000..86400000). The synchronous reservation lives in selection state and resets on proxy restart or selection-state reset; in-flight attempted targets are never reopened. New failures record cooldowns normally, and provider quota, account admission, payload eligibility, metered-spend and cancellation checks remain authoritative. There is no timer; existing send bounds remain. The opt-in may retry a target before its server Retry-After. CLI `--cooldown-reprobe on|off` and `--cooldown-reprobe-interval` control it; omitted management fields preserve it on unrelated saves. Tests: `tests/codex-integration/combo-cooldown-reprobe.test.ts` and `tests/cli/cli-combo-parity.test.ts`. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. ## Combo default effort precedence diff --git a/tests/cli/cli-combo-parity.test.ts b/tests/cli/cli-combo-parity.test.ts index 73fbd337dd3..28f2936092c 100644 --- a/tests/cli/cli-combo-parity.test.ts +++ b/tests/cli/cli-combo-parity.test.ts @@ -136,6 +136,38 @@ function realRuntime(refresh: CatalogDisposition = { status: "committed", change } describe("integrated combo extension and actual management persistence", () => { + test("cooldown reprobe opt-in, interval, carry and explicit off survive persistence", async () => { + const f = realRuntime(); + expect(await handleComboCommand(["set", "saved", "--cooldown-reprobe", "on", "--cooldown-reprobe-interval", "120000", "--json"], f.deps)).toBe(0); + expect(f.config.combos!.saved!.cooldownReprobe).toEqual({ enabled: true, intervalMs: 120000 }); + expect(await handleComboCommand(["set", "saved", "--display-name", "Retained", "--json"], f.deps)).toBe(0); + expect(f.config.combos!.saved!.cooldownReprobe).toEqual({ enabled: true, intervalMs: 120000 }); + // A GUI or older API client omitting the field also carries the saved opt-in. + const response = await handleComboRoutes({ + req: new Request("http://localhost/api/combos", { method: "PUT", body: JSON.stringify({ + id: "saved", combo: { targets: [target], strategy: "failover", defaultEffort: "high" }, + }) }), url: new URL("http://localhost/api/combos"), config: f.config, deps: {}, version: "fixture", + trustedLoopbackIngress: true, guiSessionIssuance: null, + convergeCodexCatalog: async () => ({ status: "committed", changed: false, degraded: false, notices: [] }), + syncClaudeAgentDefsBestEffort: async () => {}, + }); + expect(response!.status).toBe(200); + expect(f.config.combos!.saved!.cooldownReprobe?.enabled).toBe(true); + expect(await handleComboCommand(["set", "saved", "--cooldown-reprobe", "off", "--json"], f.deps)).toBe(0); + expect(f.config.combos!.saved!.cooldownReprobe).toBeUndefined(); + }); + test("interval alone does not enable cooldown recovery", async () => { + const f = realRuntime(); + expect(await handleComboCommand(["set", "saved", "--cooldown-reprobe-interval", "1000", "--json"], f.deps)).toBe(0); + expect(f.config.combos!.saved!.cooldownReprobe).toBeUndefined(); + }); + test("invalid cooldown reprobe flags refuse before contacting the proxy", async () => { + for (const args of [["--cooldown-reprobe", "yes"], ["--cooldown-reprobe-interval", "0"], ["--cooldown-reprobe-interval", "1.5"], ["--cooldown-reprobe-interval", "86400001"]]) { + const f = realRuntime(); + expect(await handleComboCommand(["set", "saved", ...args], f.deps)).toBe(2); + expect(f.calls).toEqual([]); + } + }); test("a target-file replacement preserves unrelated strategy and public identity", async () => { const f = realRuntime(); Object.assign(f.config.combos!.saved!, { strategy: "round-robin", stickyLimit: 3, alias: "my-combo", displayName: "My combo" }); diff --git a/tests/codex-integration/combo-cooldown-reprobe.test.ts b/tests/codex-integration/combo-cooldown-reprobe.test.ts new file mode 100644 index 00000000000..acdebbbdf33 --- /dev/null +++ b/tests/codex-integration/combo-cooldown-reprobe.test.ts @@ -0,0 +1,116 @@ +import { describe, test, expect } from 'bun:test'; +import { pickComboTarget, pickComboTargetWithWait, advanceComboAfterFailure, clearComboSelectionState } from '../../src/combos/resolve'; +import { coolComboTarget, isComboTargetInCooldown } from '../../src/combos/failover'; +import { comboConfigIssues, normalizeComboConfig } from '../../src/combos/types'; +import { routeModel } from '../../src/router'; +import type { OcxConfig, OcxComboConfig } from '../../src/types'; + +const targets = [{ provider: 'openai', model: 'test-a', weight: 1 }, { provider: 'openai', model: 'test-b', weight: 1 }]; +const now = 1_000_000; +let sequence = 0; +function setup(reprobe: OcxComboConfig['cooldownReprobe'] = { enabled: true }) { + const id = `reprobe-${++sequence}`; + const config = { providers: { openai: { baseUrl: 'https://example.invalid', apiKey: 'test' } }, + combos: { [id]: { strategy: 'failover', targets, ...(reprobe ? { cooldownReprobe: reprobe } : {}) } } } as OcxConfig; + cool(id); + return { id, config }; +} +function cool(id: string, time = now) { + for (const target of targets) coolComboTarget(id, target, { now: time, retryAfter: '36000', status: 429 }); +} + +describe('opt-in cooldown recovery', () => { + test('omitted and explicitly disabled preserve current behavior', () => { + for (const value of [undefined, { enabled: false }]) { + const { id, config } = setup(value); + if (!value) delete config.combos![id]!.cooldownReprobe; + expect(pickComboTarget(config, id, { now })).toBeNull(); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); + } + }); + test('one ordered pass; errors re-cool; no second pass before default interval', () => { + const { id, config } = setup(); + const first = pickComboTarget(config, id, { now })!; + expect(first.target.model).toBe('test-a'); + const second = advanceComboAfterFailure(config, first, { now, status: 429, retryAfter: '36000', cooldownScope: 'target' })!; + expect(second.target.model).toBe('test-b'); + expect(advanceComboAfterFailure(config, second, { now, status: 402, retryAfter: '36000', cooldownScope: 'target' })).toBeNull(); + expect(pickComboTarget(config, id, { now: now + 299999 })).toBeNull(); + expect(pickComboTarget(config, id, { now: now + 300000 })!.target.model).toBe('test-a'); + }); + test('interval is configurable and reservations are isolated by combo', () => { + const a = setup({ enabled: true, intervalMs: 1000 }); + const b = setup({ enabled: true, intervalMs: 1000 }); + expect(pickComboTarget(a.config, a.id, { now })).not.toBeNull(); + cool(a.id); + expect(pickComboTarget(a.config, a.id, { now: now + 999 })).toBeNull(); + expect(pickComboTarget(b.config, b.id, { now })).not.toBeNull(); + expect(pickComboTarget(a.config, a.id, { now: now + 1000 })).not.toBeNull(); + }); + test('preview and in-flight exclusions do not reopen targets', () => { + const { id, config } = setup(); + expect(pickComboTarget(config, id, { now, preview: true })).toBeNull(); + expect(pickComboTarget(config, id, { now, exclude: ['openai/test-a'] })).toBeNull(); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); + }); + test('available targets prevent a reset of siblings', () => { + const { id, config } = setup(); + pickComboTarget(config, id, { now }); + coolComboTarget(id, targets[0]!, { now, retryAfter: '36000', status: 402 }); + expect(pickComboTarget(config, id, { now })!.target.model).toBe('test-b'); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); + }); + test('wait path reserves synchronously across concurrent callers', async () => { + const { id, config } = setup(); + const first = await pickComboTargetWithWait(config, id, { now, waitForCooldownMs: 0 }); + expect(first!.target.model).toBe('test-a'); + cool(id); + const results = await Promise.all(Array.from({ length: 4 }, () => + pickComboTargetWithWait(config, id, { now, waitForCooldownMs: 0 }))); + expect(results).toEqual([null, null, null, null]); + }); + test('eligibility, disabled providers, and cancellation remain enforced', async () => { + const { id, config } = setup(); + expect(pickComboTarget(config, id, { now, eligible: () => false })).toBeNull(); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); + const abort = new AbortController(); abort.abort(); + expect(await pickComboTargetWithWait(config, id, { now, waitForCooldownMs: 0, abortSignal: abort.signal })).toBeNull(); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); + config.providers.openai!.disabled = true; + expect(pickComboTarget(config, id, { now })).toBeNull(); + }); + test('selection-state reset releases reservation', () => { + const { id, config } = setup(); + pickComboTarget(config, id, { now }); cool(id); + clearComboSelectionState(id); + expect(pickComboTarget(config, id, { now })).not.toBeNull(); + }); + test('router can recover an all-cooled combo, while preview is side-effect-free', () => { + const { id, config } = setup(); + // Router uses the real clock; keep these cooldowns ahead of it. + cool(id, Date.now()); + const route = routeModel(config, `combo/${id}`); + expect(route.providerName).toBe('openai'); + expect(route.modelId).toBe('test-a'); + }); +}); + +describe('configuration validation', () => { + test('defaults to disabled, and enabled defaults to five minutes', () => { + expect(normalizeComboConfig({ targets }).cooldownReprobe).toBeUndefined(); + expect(normalizeComboConfig({ targets, cooldownReprobe: { enabled: false } }).cooldownReprobe).toBeUndefined(); + expect(normalizeComboConfig({ targets, cooldownReprobe: { enabled: true } }).cooldownReprobe).toEqual({ enabled: true, intervalMs: 300000 }); + }); + test('accepts interval bounds and rejects invalid/unknown settings', () => { + const providers = { openai: { baseUrl: 'https://example.invalid' } }; + for (const intervalMs of [1000, 86400000]) { + expect(comboConfigIssues('valid', { targets, cooldownReprobe: { enabled: true, intervalMs } }, providers)).toEqual([]); + } + for (const cooldownReprobe of [null, [], true, {}, { enabled: 'yes' }, + { enabled: true, intervalMs: 0 }, { enabled: true, intervalMs: 1000.5 }, + { enabled: true, intervalMs: 86400001 }, { enabled: true, intervalMs: '300000' }, + { enabled: true, extra: true }]) { + expect(comboConfigIssues('invalid', { targets, cooldownReprobe }, providers).some(i => i.path[0] === 'cooldownReprobe')).toBe(true); + } + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index d41fccb67a1..c44f757a46f 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -603,6 +603,7 @@ "coding-agent-tool-result-images.test.ts": "adapters", "cold-spawn-warmup.test.ts": "ci-workflows", "combo-authoritative-reset.test.ts": "codex-integration", "combo-codex-exhaustion-cooldown.test.ts": "codex-integration", + "combo-cooldown-reprobe.test.ts": "codex-integration", "combo-permanent-failure-cooldown.test.ts": "codex-integration", "combo-reset-window-model-scope.test.ts": "codex-integration", "combo-child-headers.test.ts": "routing", "combo-last-resort.test.ts": "codex-integration", "combo-management-api.test.ts": "routing", From 73a3cfc34f6c656afb80dfb250c9001480d0fbfc Mon Sep 17 00:00:00 2001 From: Artur Safonov Date: Sun, 25 Oct 2026 17:02:21 +0200 Subject: [PATCH 2/5] fix(ci): shard combo test registration and refresh skill surface --- scripts/test-layout/layout-combos.json | 3 +++ scripts/test-layout/layout.json | 3 +-- skills/ocx/references/01_surface_agents-routing.md | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 scripts/test-layout/layout-combos.json diff --git a/scripts/test-layout/layout-combos.json b/scripts/test-layout/layout-combos.json new file mode 100644 index 00000000000..9faa45d117b --- /dev/null +++ b/scripts/test-layout/layout-combos.json @@ -0,0 +1,3 @@ +{ + "combo-cooldown-reprobe.test.ts": "codex-integration" +} diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index b200958a8c6..774f2380102 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1,7 +1,7 @@ { "version": 1, "root": "tests", - "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json"], + "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json", "layout-combos.json"], "explicit": { "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "codex-provider-routing.test.ts": "codex-integration", "config-native-reasoning-retention.test.ts": "config", "server-native-reasoning-retention-merge.test.ts": "server", "server-native-reasoning-retention.test.ts": "server", "responses-retained-reasoning-stream-recovery.test.ts": "responses", "openai-native-hosted-search-history.test.ts": "adapters/openai", "responses-native-compact-search-history.test.ts": "responses", "config-hosted-search-projection.test.ts": "config", "gcp-credential-boundary.test.ts": "adapters/google", "gcp-sa-marker.test.ts": "adapters/google", "gcp-sa-adapter.test.ts": "adapters/google", "management-gcp-credential-input.test.ts": "server", "codex-preserve-signed-in.test.ts": "codex-integration", "cli-account-preservation.test.ts": "cli", "cli-provider-list-active.test.ts": "cli", "service-state-isolation.test.ts": "service", "compaction-history-framing.test.ts": "responses", "catalog-native-max-context.test.ts": "codex-integration", "protocol-argon-thought-boundary.test.ts": "responses", "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "bridge-terminal-refusal.test.ts": "adapters", "loop-terminal-refusal.test.ts": "images", "web-search-terminal-refusal.test.ts": "web-search", "combo-terminal-refusal.test.ts": "routing", "server-terminal-refusal-wiring.test.ts": "server", "incomplete-reason-normalization.test.ts": "responses", "advisor-retention.test.ts": "advisor", "responses-native-key-cooldown.test.ts": "responses", "responses-run-turn-key-readmission.test.ts": "responses", "qoder-spawn-readmission.test.ts": "adapters", "api-key-selection-cooldown.test.ts": "providers", "api-key-auto-pause-distribution.test.ts": "providers", "explicit-key-pause-oauth-boundary.test.ts": "providers", "server-chat-native-key-selection.test.ts": "server", "compact-key-selection.test.ts": "responses", "messages-native-key-selection.test.ts": "claude-integration", "coding-agent-windows-dispatch.test.ts": "adapters", "coding-agent-native-empty-argv.test.ts": "windows", "service-proxy-env.test.ts": "service", "openai-chat-standalone-bare-block.test.ts": "adapters/openai", "owner-registry-isolation.test.ts": "config", @@ -741,7 +741,6 @@ "cold-spawn-warmup.test.ts": "ci-workflows", "combo-authoritative-reset.test.ts": "codex-integration", "combo-codex-exhaustion-cooldown.test.ts": "codex-integration", - "combo-cooldown-reprobe.test.ts": "codex-integration", "combo-permanent-failure-cooldown.test.ts": "codex-integration", "combo-reset-window-model-scope.test.ts": "codex-integration", "combo-child-headers.test.ts": "routing", diff --git a/skills/ocx/references/01_surface_agents-routing.md b/skills/ocx/references/01_surface_agents-routing.md index 48c95c7c68f..27f730badc9 100644 --- a/skills/ocx/references/01_surface_agents-routing.md +++ b/skills/ocx/references/01_surface_agents-routing.md @@ -958,7 +958,7 @@ State-changing: no. JSON mode: `payload`. ### `ocx combo set` -Usage: `ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] [--effort ] [--effort-mode ] [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] [--description ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] [--session-pin ] [--rename-from ] [--image-input ] [--reasoning-effort-mode ] [--json]` +Usage: `ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] [--cooldown-reprobe ] [--cooldown-reprobe-interval ] [--effort ] [--effort-mode ] [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] [--description ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] [--session-pin ] [--rename-from ] [--image-input ] [--reasoning-effort-mode ] [--json]` Create, update or rename a combo. @@ -975,6 +975,8 @@ State-changing: yes. JSON mode: `payload`. | `--targets` | string | Ordered provider/model[:weight] list; required for creation. | | `--strategy` | string | Routing strategy. | | `--sticky` | string | Round-robin limit 1-100; - resets. | +| `--cooldown-reprobe` | string | Opt into one recovery pass when all eligible targets cool; off by default. | +| `--cooldown-reprobe-interval` | string | Minimum interval between recovery passes in ms (1000..86400000); default 300000. | | `--effort` | string | Default effort; - clears. | | `--effort-mode` | string | fallback or force; - resets to fallback. | | `--alias` | string | Alias; - clears. | From 62aa6536952428ac2e5e1683776437bea278d2b1 Mon Sep 17 00:00:00 2001 From: Artur Safonov Date: Sun, 25 Oct 2026 17:19:27 +0200 Subject: [PATCH 3/5] fix(combos): scope cooldown recovery to eligible initial picks --- src/combos/failover.ts | 7 ++- src/combos/resolve.ts | 16 ++++-- src/server/responses/core-combo.ts | 6 ++- structure/fallback-policy.md | 2 +- structure/providers-and-adapters.md | 2 +- structure/runtime.md | 2 +- .../combo-cooldown-reprobe.test.ts | 53 +++++++++++++++++-- .../server-combo-cooldown-recording.test.ts | 20 +++++++ 8 files changed, 93 insertions(+), 15 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index bf590e6ee0b..1041f7ef037 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -313,7 +313,8 @@ export function sweepExpiredComboTargetCooldowns(now = Date.now()): number { return removed; } -export function clearComboTargetCooldowns(comboId?: string): void { +/** Clear all cooldowns, one combo, or only the supplied target keys within that combo. */ +export function clearComboTargetCooldowns(comboId?: string, targetKeys?: Iterable): void { if (comboId === undefined) { targetCooldowns.clear(); liveComboTargets.clear(); @@ -321,6 +322,10 @@ export function clearComboTargetCooldowns(comboId?: string): void { return; } const prefix = `${comboId}\0`; + if (targetKeys !== undefined) { + for (const key of targetKeys) targetCooldowns.delete(`${prefix}${key}`); + return; + } for (const key of targetCooldowns.keys()) { if (key.startsWith(prefix)) targetCooldowns.delete(key); } diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index c8cd848bf5c..200c98f056f 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -305,7 +305,7 @@ function reopenAllCooledTargets( const next = state ?? { successes: 0, currentWeights: new Map(), successfulUses: new Map() }; next.lastCooldownReprobeAt = now; selectionState.set(comboId, next); - clearComboTargetCooldowns(comboId); + clearComboTargetCooldowns(comboId, usable.map(targetKey)); console.warn(`[combo] ${comboId}: reopening all-cooled targets; minimum reprobe interval ${combo.cooldownReprobe.intervalMs}ms`); } @@ -315,6 +315,8 @@ export function pickComboTarget( options: { exclude?: Iterable; eligible?: (target: NormalizedComboTarget) => boolean; + /** Disable recovery for picks without request context or picks already in flight. */ + allowCooldownReprobe?: boolean; now?: number; /** Inspect a round-robin choice without mutating its sticky/weight state. */ preview?: boolean; @@ -325,7 +327,7 @@ export function pickComboTarget( if (!combo) throw new UnknownComboError(comboId); const excluded = new Set(options.exclude ?? []); const now = options.now ?? Date.now(); - if (!options.preview && excluded.size === 0) { + if (options.allowCooldownReprobe !== false && !options.preview && excluded.size === 0) { reopenAllCooledTargets(config, comboId, now, options.eligible); } const eligible = (target: NormalizedComboTarget): boolean => @@ -524,6 +526,8 @@ export async function pickComboTargetWithWait( eligible?: (target: NormalizedComboTarget) => boolean; waitForCooldownMs: number; abortSignal?: AbortSignal; + /** Disable recovery when retrying a target after its normal cooldown. */ + allowCooldownReprobe?: boolean; now?: number; sleep?: (ms: number, signal?: AbortSignal) => Promise; }, @@ -531,7 +535,7 @@ export async function pickComboTargetWithWait( const now = options.now ?? Date.now(); const excluded = new Set(options.exclude ?? []); const customEligible = options.eligible; - if (excluded.size === 0 && !options.abortSignal?.aborted) { + if (options.allowCooldownReprobe !== false && excluded.size === 0 && !options.abortSignal?.aborted) { reopenAllCooledTargets(config, comboId, now, customEligible); } const eligibleAt = (target: NormalizedComboTarget, at: number): boolean => @@ -558,6 +562,7 @@ export async function pickComboTargetWithWait( const normalOnly = (target: NormalizedComboTarget): boolean => !target.lastResort && eligible(target); const normalPick = pickComboTarget(config, comboId, { + allowCooldownReprobe: false, exclude: excluded, eligible: normalOnly, now, @@ -588,6 +593,7 @@ export async function pickComboTargetWithWait( // The combo can be deleted or renamed while this request sleeps. if (!getCombo(config, comboId)) return null; const waited = pickComboTarget(config, comboId, { + allowCooldownReprobe: false, exclude: excluded, now: now + normalDelay, eligible: target => !target.lastResort @@ -605,6 +611,7 @@ export async function pickComboTargetWithWait( const remainingWaitMs = options.waitForCooldownMs - spentWaitMs; const pick = pickComboTarget(config, comboId, { + allowCooldownReprobe: false, exclude: excluded, eligible: target => eligibleAt(target, clock), now: clock, @@ -639,6 +646,7 @@ export async function pickComboTargetWithWait( if (!getCombo(config, comboId)) return null; return pickComboTarget(config, comboId, { exclude: excluded, + allowCooldownReprobe: false, now: clock + delay, eligible: targetCandidate => eligibleAt(targetCandidate, clock + delay), }); @@ -688,7 +696,7 @@ export function tryPickComboModel(config: OcxConfig, modelId: string, preview = const comboId = resolveComboId(config, modelId); if (!comboId) return null; if (!getCombo(config, comboId)) throw new UnknownComboError(comboId); - const picked = pickComboTarget(config, comboId, { preview }); + const picked = pickComboTarget(config, comboId, { preview, allowCooldownReprobe: false }); if (!picked) throw new NoAvailableComboTargetsError(comboId, comboExclusionReasons(config, comboId)); return picked; } diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 8c801131992..6c935e2f14e 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -582,6 +582,7 @@ export async function executeComboResponses( const pickWithWait = (pickOptions: { exclude?: Iterable; eligible?: (target: NonNullable["targets"][number]) => boolean; + allowCooldownReprobe?: boolean; now?: number; }) => pickComboTargetWithWait(config, comboId, { ...pickOptions, @@ -1362,10 +1363,11 @@ export async function executeComboResponses( const failedTargetCooled = failureCooldownScope !== "none" && failedTargetCooldownRecorded && isComboTargetInCooldown(comboId, pick.target, failureNow); - // Same target selector as the exclusionary pick below, minus `exclude`: the only - // difference is deliberate and is the whole point of the single-target retry. + // Drop exclusions for the single-target retry, but wait for normal cooldown + // expiry: an in-flight request cannot reserve a fresh recovery pass. const retryAfterCooldown = () => pickWithWait({ + allowCooldownReprobe: false, eligible: targetEligible, now: failureNow, }); diff --git a/structure/fallback-policy.md b/structure/fallback-policy.md index ae36c897249..f91721d3fb0 100644 --- a/structure/fallback-policy.md +++ b/structure/fallback-policy.md @@ -130,4 +130,4 @@ privacy), `tests/responses/responses-combo-fallback-off.test.ts`, `tests/routing `tests/server/fallback-routes.test.ts`, `tests/clients/client-home-unreachable.test.ts`, `tests/cli/cli-fallback.test.ts` and `gui/tests/fallback-panel.test.tsx`. -The opt-in [all-cooled Combo recovery](runtime.md#scoped-provider-quota-for-combo-selection) can reopen saved combo cooldowns in `src/combos/resolve.ts`; it does not authorize a fallback hop or bypass the configured fallback policy. +The opt-in [all-cooled Combo recovery](runtime.md#scoped-provider-quota-for-combo-selection) can reopen saved combo cooldowns in `src/combos/resolve.ts`; only the request-eligible target keys are cleared. Route-level picks and in-flight retry-after-cooldown selections disable reprobes. Recovery does not authorize a fallback hop or bypass the configured fallback policy. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 3ebcbeb8246..efec9bb9b76 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -589,7 +589,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr remains authoritative. No raw body is retained in that field, serialized into client errors, or logged. Presence of both `detail` and `error` across the original root and its own non-array `response` record is ambiguous, regardless of values; null counts and deeper quoted objects do not. This is checked before selecting an error carrier. For complete bounded HTTP errors, the text wrapper selects root `detail`/`error` when present; otherwise it selects the own `response` record for the same exact `detail` or `error.message` match. It never searches another carrier after a nonmatch or descends further. Policy-fallback and direct callers share this wrapper; SSE message parsing disables nested HTTP selection. The same bounded HTTP parser retains allowlisted dedicated `code` fields from root and response before display truncation; diagnostic `type` is never promoted, including when code is absent or malformed. Policy/direct classification inspects full text only without authoritative carrier metadata; SSE never borrows hard codes from quoted message JSON. -Committed output, cancellation, hard policy refusals and resend limits remain stronger than model fallback. Opt-in all-cooled target recovery in `src/combos/resolve.ts` follows [the Combo cooldown contract](runtime.md#scoped-provider-quota-for-combo-selection) without widening account/provider admission. +Committed output, cancellation, hard policy refusals and resend limits remain stronger than model fallback. Opt-in all-cooled target recovery in `src/combos/resolve.ts` follows [the Combo cooldown contract](runtime.md#scoped-provider-quota-for-combo-selection) without widening account/provider admission. It clears only request-eligible target cooldowns; route-level selection and in-flight cooldown retries cannot initiate a reprobe. Token Plan dated-model policy (2026-10-20): both curated presets include exact `deepseek-v4.1-flash-1017` metadata without changing live discovery or Responses pins. diff --git a/structure/runtime.md b/structure/runtime.md index feaae862186..a6d059cd88f 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. Every combo, routing-profile, subagent and hard-lock hop to another model or provider first passes the [fallback policy](fallback-policy.md#sites). -`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. Opt-in `cooldownReprobe: { enabled: true, intervalMs }` in `src/combos/resolve.ts` permits an initial non-preview request to clear all-cooled eligible targets once per configured interval (default 300000 ms, range 1000..86400000). The synchronous reservation lives in selection state and resets on proxy restart or selection-state reset; in-flight attempted targets are never reopened. New failures record cooldowns normally, and provider quota, account admission, payload eligibility, metered-spend and cancellation checks remain authoritative. There is no timer; existing send bounds remain. The opt-in may retry a target before its server Retry-After. CLI `--cooldown-reprobe on|off` and `--cooldown-reprobe-interval` control it; omitted management fields preserve it on unrelated saves. Tests: `tests/codex-integration/combo-cooldown-reprobe.test.ts` and `tests/cli/cli-combo-parity.test.ts`. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. +`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. Opt-in `cooldownReprobe: { enabled: true, intervalMs }` in `src/combos/resolve.ts` permits an initial non-preview request to clear all-cooled eligible targets once per configured interval (default 300000 ms, range 1000..86400000). The synchronous reservation lives in selection state and resets on proxy restart or selection-state reset; Only target keys eligible for the current request are cleared; other target cooldowns are preserved. Route-level selection without request eligibility disables reprobes, as does the in-flight retry-after-cooldown path, which waits for normal expiry. Nested wait-path selections cannot start a second reprobe. New failures record cooldowns normally, and provider quota, account admission, payload eligibility, metered-spend and cancellation checks remain authoritative. There is no timer; existing send bounds remain. The opt-in may retry a target before its server Retry-After. CLI `--cooldown-reprobe on|off` and `--cooldown-reprobe-interval` control it; omitted management fields preserve it on unrelated saves. Tests: `tests/codex-integration/combo-cooldown-reprobe.test.ts` and `tests/cli/cli-combo-parity.test.ts`. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. ## Combo default effort precedence diff --git a/tests/codex-integration/combo-cooldown-reprobe.test.ts b/tests/codex-integration/combo-cooldown-reprobe.test.ts index acdebbbdf33..ba2668edf17 100644 --- a/tests/codex-integration/combo-cooldown-reprobe.test.ts +++ b/tests/codex-integration/combo-cooldown-reprobe.test.ts @@ -1,6 +1,6 @@ import { describe, test, expect } from 'bun:test'; import { pickComboTarget, pickComboTargetWithWait, advanceComboAfterFailure, clearComboSelectionState } from '../../src/combos/resolve'; -import { coolComboTarget, isComboTargetInCooldown } from '../../src/combos/failover'; +import { clearComboTargetCooldowns, coolComboTarget, isComboTargetInCooldown } from '../../src/combos/failover'; import { comboConfigIssues, normalizeComboConfig } from '../../src/combos/types'; import { routeModel } from '../../src/router'; import type { OcxConfig, OcxComboConfig } from '../../src/types'; @@ -60,6 +60,47 @@ describe('opt-in cooldown recovery', () => { expect(pickComboTarget(config, id, { now })!.target.model).toBe('test-b'); expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); }); + test('recovery preserves cooldowns for targets ineligible for this request', () => { + const { id, config } = setup(); + const pick = pickComboTarget(config, id, { now, eligible: target => target.model === 'test-a' }); + expect(pick!.target.model).toBe('test-a'); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(false); + expect(isComboTargetInCooldown(id, targets[1]!, now)).toBe(true); + }); + test('scoped cooldown clearing preserves other targets and combos', () => { + const a = setup(); + const b = setup(); + clearComboTargetCooldowns(a.id, []); + expect(isComboTargetInCooldown(a.id, targets[0]!, now)).toBe(true); + clearComboTargetCooldowns(a.id, ['openai/test-a']); + expect(isComboTargetInCooldown(a.id, targets[0]!, now)).toBe(false); + expect(isComboTargetInCooldown(a.id, targets[1]!, now)).toBe(true); + expect(isComboTargetInCooldown(b.id, targets[0]!, now)).toBe(true); + clearComboTargetCooldowns(a.id); + expect(isComboTargetInCooldown(a.id, targets[1]!, now)).toBe(false); + expect(isComboTargetInCooldown(b.id, targets[1]!, now)).toBe(true); + }); + test('explicit opt-out preserves cooldowns without consuming a recovery reservation', async () => { + const { id, config } = setup(); + expect(pickComboTarget(config, id, { now, allowCooldownReprobe: false })).toBeNull(); + expect(await pickComboTargetWithWait(config, id, { + now, waitForCooldownMs: 0, allowCooldownReprobe: false, + })).toBeNull(); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); + expect(await pickComboTargetWithWait(config, id, { now, waitForCooldownMs: 0 })).not.toBeNull(); + }); + test('in-flight retries wait for normal cooldown expiry even after the reprobe interval', async () => { + const { id, config } = setup({ enabled: true, intervalMs: 1000 }); + expect(pickComboTarget(config, id, { now })).not.toBeNull(); + for (const target of targets) coolComboTarget(id, target, { now, retryAfter: '2', status: 429 }); + const sleeps: number[] = []; + const pick = await pickComboTargetWithWait(config, id, { + now: now + 1000, waitForCooldownMs: 1000, allowCooldownReprobe: false, + sleep: async delay => { sleeps.push(delay); }, + }); + expect(sleeps).toEqual([1000]); + expect(pick!.target.model).toBe('test-a'); + }); test('wait path reserves synchronously across concurrent callers', async () => { const { id, config } = setup(); const first = await pickComboTargetWithWait(config, id, { now, waitForCooldownMs: 0 }); @@ -85,13 +126,15 @@ describe('opt-in cooldown recovery', () => { clearComboSelectionState(id); expect(pickComboTarget(config, id, { now })).not.toBeNull(); }); - test('router can recover an all-cooled combo, while preview is side-effect-free', () => { + test('route-level selection cannot reopen targets without request eligibility', () => { const { id, config } = setup(); // Router uses the real clock; keep these cooldowns ahead of it. cool(id, Date.now()); - const route = routeModel(config, `combo/${id}`); - expect(route.providerName).toBe('openai'); - expect(route.modelId).toBe('test-a'); + expect(() => routeModel(config, `combo/${id}`)).toThrow(); + expect(isComboTargetInCooldown(id, targets[0]!, Date.now())).toBe(true); + expect(isComboTargetInCooldown(id, targets[1]!, Date.now())).toBe(true); + expect(pickComboTarget(config, id, { now: Date.now(), eligible: target => target.model === 'test-a' })).not.toBeNull(); + expect(isComboTargetInCooldown(id, targets[1]!, Date.now())).toBe(true); }); }); diff --git a/tests/server/server-combo-cooldown-recording.test.ts b/tests/server/server-combo-cooldown-recording.test.ts index 337244a62fa..7fb4b9f838d 100644 --- a/tests/server/server-combo-cooldown-recording.test.ts +++ b/tests/server/server-combo-cooldown-recording.test.ts @@ -95,6 +95,26 @@ afterEach(async () => { removeTreeWithRetry(home); } }); +test("an in-flight cooldown retry cannot initiate an opt-in reprobe", async () => { + let hits = 0; + const server = serve(() => { + hits += 1; + return hits === 1 + ? Response.json({ error: { message: "rate limited" } }, { + status: 429, headers: { "retry-after": "2" }, + }) + : chatSuccess("unexpected immediate retry", "m1"); + }); + const cfg = comboConfig({ a: provider("openai-chat", baseUrl(server), "synthetic-key") }, [target], { + cooldownReprobe: { enabled: true, intervalMs: 1000 }, waitForCooldownMs: 100, + }); + const response = await post(cfg); + await response.text(); + expect(response.status).toBe(429); + expect(hits).toBe(1); + expect(isComboTargetInCooldown("free", target)).toBe(true); +}); + test("cooldown recording distinguishes a successful write from a stale removed writer", () => { expect(coolComboTarget("free", target, { cooldownMs: 1000 })).toBe(true); reconcileComboTargetCooldowns(removalContext()); From 8770b734e47c729de1a5e14fa893927f3b4a2220 Mon Sep 17 00:00:00 2001 From: Artur Safonov Date: Sun, 25 Oct 2026 17:40:43 +0200 Subject: [PATCH 4/5] fix(combos): preserve eligibility across recovery selection --- src/combos/resolve.ts | 11 ++++- src/server/responses/core-combo.ts | 12 ++++-- src/server/responses/request-prepare.ts | 5 ++- structure/fallback-policy.md | 2 +- structure/providers-and-adapters.md | 2 +- structure/runtime.md | 2 +- .../combo-cooldown-reprobe.test.ts | 29 +++++++++++++ .../server/agent-task-recovery-combo.test.ts | 42 ++++++++++++++++++- tests/server/combo-metered-consent.test.ts | 2 +- 9 files changed, 95 insertions(+), 12 deletions(-) diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index 200c98f056f..60d07e6ba61 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -298,8 +298,15 @@ function reopenAllCooledTargets( const state = selectionState.get(comboId); if (state?.lastCooldownReprobeAt !== undefined && now - state.lastCooldownReprobeAt < combo.cooldownReprobe.intervalMs) return; - const usable = combo.targets.filter(target => targetProviderIsUsable(config, target, now) - && (eligible?.(target) ?? true)); + const reprobeEligible = (target: NormalizedComboTarget): boolean => { + if (!targetProviderIsUsable(config, target, now)) return false; + try { return eligible?.(target) ?? true; } catch { return false; } + }; + // Ordinary selection skips cooled targets without evaluating their request predicate. + // Preserve that behavior when a sibling is already available. + if (combo.targets.some(target => !isComboTargetInCooldown(comboId, target, now) + && reprobeEligible(target))) return; + const usable = combo.targets.filter(reprobeEligible); if (usable.length === 0 || !usable.every(target => isComboTargetInCooldown(comboId, target, now))) return; // No await between reservation and reset: concurrent requests cannot reserve another pass. const next = state ?? { successes: 0, currentWeights: new Map(), successfulUses: new Map() }; diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 6c935e2f14e..6badc29d8bd 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -509,12 +509,14 @@ export async function executeComboResponses( const payloadEligible = (target: (typeof combo.targets)[number]): boolean => comboPayloadReadable || !unreadableEncryptedAgentTask || canDecryptUnreadableAgentTask(target); const meteredAllowed = admissionMayUseMeteredComboTargets(config, options.admission, combo); - const targetEligible = (target: (typeof combo.targets)[number]): boolean => + // Recovery can replace unreadable encrypted input, but cannot relax other admission checks. + const recoveryEligible = (target: (typeof combo.targets)[number]): boolean => (meteredAllowed || !target.metered) && (combo.strategy !== "jev" || target.provider !== JEV_PROVIDER_ID) - && payloadEligible(target) && reasoningReplayEligible(target) && (protocolLanes?.pickable(target) ?? true); + const targetEligible = (target: (typeof combo.targets)[number]): boolean => + payloadEligible(target) && recoveryEligible(target); const onlyReplayIncompatibleTargetsRemain = (excluded: Iterable = []): boolean => { const excludedKeys = new Set(excluded); const remaining = combo.targets.filter(target => { @@ -614,7 +616,7 @@ export async function executeComboResponses( : jevSessionPinScope({ combo, comboId, admission: options.admission, lane: sessionLaneIdFromRequest(req.headers) }); if (unreadableEncryptedAgentTask && !pick) { - pick = await pickWithWait({ now: initialNow }); + pick = await pickWithWait({ eligible: recoveryEligible, now: initialNow }); if (!pick) { discardEncryptedAgentTaskRecovery( req, @@ -625,7 +627,7 @@ export async function executeComboResponses( return options.abortSignal?.aborted ? clientCancelledResponse() : comboUnavailable(comboId, undefined, - comboExclusionReasons(config, comboId, target => meteredAllowed || !target.metered)); + comboExclusionReasons(config, comboId, recoveryEligible)); } if (!(await recoverUnreadableEncryptedTask())) { return options.abortSignal?.aborted @@ -1275,6 +1277,7 @@ export async function executeComboResponses( const recoveredTarget = await pickWithWait({ exclude: pick.attempted, eligible: target => { + if (!recoveryEligible(target)) return false; try { const route = routeConcreteModel(config, `${target.provider}/${target.model}`); return route.codexAccountMode === undefined @@ -1409,6 +1412,7 @@ export async function executeComboResponses( if (unreadableEncryptedAgentTask && !comboPayloadReadable) { const recoveredTarget = await pickWithWait({ exclude: attemptedTargets, + eligible: recoveryEligible, now: failureNow, }); if (recoveredTarget && await recoverUnreadableEncryptedTask()) { diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index dbd8496f39d..8a56b7abdd6 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -665,7 +665,10 @@ export async function prepareResponsesRequest( if (candidate.combo?.target.metered && !admissionMayUseMeteredComboTargets( config, options.admission, config.combos?.[candidate.combo.comboId], )) { - const pick = pickComboTarget(config, candidate.combo.comboId, { eligible: target => !target.metered }); + const pick = pickComboTarget(config, candidate.combo.comboId, { + eligible: target => !target.metered, + allowCooldownReprobe: false, + }); if (!pick) { throw new NoAvailableComboTargetsError(candidate.combo.comboId, comboExclusionReasons(config, candidate.combo.comboId, target => !target.metered)); diff --git a/structure/fallback-policy.md b/structure/fallback-policy.md index f91721d3fb0..ac3c1f0ff10 100644 --- a/structure/fallback-policy.md +++ b/structure/fallback-policy.md @@ -130,4 +130,4 @@ privacy), `tests/responses/responses-combo-fallback-off.test.ts`, `tests/routing `tests/server/fallback-routes.test.ts`, `tests/clients/client-home-unreachable.test.ts`, `tests/cli/cli-fallback.test.ts` and `gui/tests/fallback-panel.test.tsx`. -The opt-in [all-cooled Combo recovery](runtime.md#scoped-provider-quota-for-combo-selection) can reopen saved combo cooldowns in `src/combos/resolve.ts`; only the request-eligible target keys are cleared. Route-level picks and in-flight retry-after-cooldown selections disable reprobes. Recovery does not authorize a fallback hop or bypass the configured fallback policy. +The opt-in [all-cooled Combo recovery](runtime.md#scoped-provider-quota-for-combo-selection) can reopen saved combo cooldowns in `src/combos/resolve.ts`; only the request-eligible target keys are cleared, with throwing predicates treated as ineligible during recovery preparation and available siblings preferred without probing cooled targets. Route-level picks and in-flight retry-after-cooldown selections disable reprobes. Recovery does not authorize a fallback hop or bypass the configured fallback policy. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index efec9bb9b76..01c977cdd88 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -589,7 +589,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr remains authoritative. No raw body is retained in that field, serialized into client errors, or logged. Presence of both `detail` and `error` across the original root and its own non-array `response` record is ambiguous, regardless of values; null counts and deeper quoted objects do not. This is checked before selecting an error carrier. For complete bounded HTTP errors, the text wrapper selects root `detail`/`error` when present; otherwise it selects the own `response` record for the same exact `detail` or `error.message` match. It never searches another carrier after a nonmatch or descends further. Policy-fallback and direct callers share this wrapper; SSE message parsing disables nested HTTP selection. The same bounded HTTP parser retains allowlisted dedicated `code` fields from root and response before display truncation; diagnostic `type` is never promoted, including when code is absent or malformed. Policy/direct classification inspects full text only without authoritative carrier metadata; SSE never borrows hard codes from quoted message JSON. -Committed output, cancellation, hard policy refusals and resend limits remain stronger than model fallback. Opt-in all-cooled target recovery in `src/combos/resolve.ts` follows [the Combo cooldown contract](runtime.md#scoped-provider-quota-for-combo-selection) without widening account/provider admission. It clears only request-eligible target cooldowns; route-level selection and in-flight cooldown retries cannot initiate a reprobe. +Committed output, cancellation, hard policy refusals and resend limits remain stronger than model fallback. Opt-in all-cooled target recovery in `src/combos/resolve.ts` follows [the Combo cooldown contract](runtime.md#scoped-provider-quota-for-combo-selection) without widening account/provider admission. It clears only request-eligible target cooldowns; route-level selection (including an unmetered reroute) and in-flight cooldown retries cannot initiate a reprobe. Encrypted-task recovery preserves request-specific JEV, reasoning-replay and protocol eligibility while relaxing payload readability. Token Plan dated-model policy (2026-10-20): both curated presets include exact `deepseek-v4.1-flash-1017` metadata without changing live discovery or Responses pins. diff --git a/structure/runtime.md b/structure/runtime.md index a6d059cd88f..56d49a3a697 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. Every combo, routing-profile, subagent and hard-lock hop to another model or provider first passes the [fallback policy](fallback-policy.md#sites). -`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. Opt-in `cooldownReprobe: { enabled: true, intervalMs }` in `src/combos/resolve.ts` permits an initial non-preview request to clear all-cooled eligible targets once per configured interval (default 300000 ms, range 1000..86400000). The synchronous reservation lives in selection state and resets on proxy restart or selection-state reset; Only target keys eligible for the current request are cleared; other target cooldowns are preserved. Route-level selection without request eligibility disables reprobes, as does the in-flight retry-after-cooldown path, which waits for normal expiry. Nested wait-path selections cannot start a second reprobe. New failures record cooldowns normally, and provider quota, account admission, payload eligibility, metered-spend and cancellation checks remain authoritative. There is no timer; existing send bounds remain. The opt-in may retry a target before its server Retry-After. CLI `--cooldown-reprobe on|off` and `--cooldown-reprobe-interval` control it; omitted management fields preserve it on unrelated saves. Tests: `tests/codex-integration/combo-cooldown-reprobe.test.ts` and `tests/cli/cli-combo-parity.test.ts`. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. +`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. Opt-in `cooldownReprobe: { enabled: true, intervalMs }` in `src/combos/resolve.ts` permits an initial non-preview request to clear all-cooled eligible targets once per configured interval (default 300000 ms, range 1000..86400000). The synchronous reservation lives in selection state and resets on proxy restart or selection-state reset; Only target keys eligible for the current request are cleared; other target cooldowns are preserved. Route-level selection without request eligibility disables reprobes, as does the in-flight retry-after-cooldown path, which waits for normal expiry. Nested wait-path selections cannot start a second reprobe. Preparation first checks available siblings without evaluating cooled targets; a throwing eligibility predicate is ineligible for recovery. Encrypted-task recovery relaxes only payload readability and retains metering, JEV, reasoning-replay and protocol-lane eligibility. New failures record cooldowns normally, and provider quota, account admission, payload eligibility, metered-spend and cancellation checks remain authoritative. There is no timer; existing send bounds remain. The opt-in may retry a target before its server Retry-After. CLI `--cooldown-reprobe on|off` and `--cooldown-reprobe-interval` control it; omitted management fields preserve it on unrelated saves. Tests: `tests/codex-integration/combo-cooldown-reprobe.test.ts` and `tests/cli/cli-combo-parity.test.ts`. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. ## Combo default effort precedence diff --git a/tests/codex-integration/combo-cooldown-reprobe.test.ts b/tests/codex-integration/combo-cooldown-reprobe.test.ts index ba2668edf17..03284daeb6a 100644 --- a/tests/codex-integration/combo-cooldown-reprobe.test.ts +++ b/tests/codex-integration/combo-cooldown-reprobe.test.ts @@ -67,6 +67,35 @@ describe('opt-in cooldown recovery', () => { expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(false); expect(isComboTargetInCooldown(id, targets[1]!, now)).toBe(true); }); + for (const wait of [false, true]) { + test(`a cooled throwing predicate cannot block an available sibling (wait=${wait})`, async () => { + const { id, config } = setup(); + clearComboTargetCooldowns(id, ['openai/test-b']); + let cooledChecks = 0; + const eligible = (target: typeof targets[number]) => { + if (target.model === 'test-a') { cooledChecks++; throw new Error('unavailable metadata'); } + return true; + }; + const pick = wait + ? await pickComboTargetWithWait(config, id, { now, eligible, waitForCooldownMs: 0 }) + : pickComboTarget(config, id, { now, eligible }); + expect(pick!.target.model).toBe('test-b'); + expect(cooledChecks).toBe(0); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); + }); + test(`reprobe treats a throwing predicate as ineligible (wait=${wait})`, async () => { + const { id, config } = setup(); + const eligible = (target: typeof targets[number]) => { + if (target.model === 'test-a') throw new Error('unavailable metadata'); + return true; + }; + const pick = wait + ? await pickComboTargetWithWait(config, id, { now, eligible, waitForCooldownMs: 0 }) + : pickComboTarget(config, id, { now, eligible }); + expect(pick!.target.model).toBe('test-b'); + expect(isComboTargetInCooldown(id, targets[0]!, now)).toBe(true); + }); + } test('scoped cooldown clearing preserves other targets and combos', () => { const a = setup(); const b = setup(); diff --git a/tests/server/agent-task-recovery-combo.test.ts b/tests/server/agent-task-recovery-combo.test.ts index 7df943326bc..4cf057ea579 100644 --- a/tests/server/agent-task-recovery-combo.test.ts +++ b/tests/server/agent-task-recovery-combo.test.ts @@ -10,7 +10,7 @@ import { } from "../../src/responses/state"; import { resetAgentTaskRecoveryState } from "../../src/server/responses/agent-task-recovery"; import { agentTaskRecoveryCacheSnapshotForTests } from "../../src/server/responses/agent-task-recovery-cache"; -import { clearComboTargetCooldowns, coolComboTarget } from "../../src/combos/failover"; +import { clearComboTargetCooldowns, coolComboTarget, isComboTargetInCooldown } from "../../src/combos/failover"; import { clearCachedProviderQuotas, setCachedProviderQuotaForTests, @@ -127,6 +127,46 @@ describe("combo path encrypted agent task recovery", () => { expect(snapshot).not.toContain(responsePayload.id!); }); + test.each([false, true])("encrypted recovery keeps replay-incompatible targets excluded (reprobe=%s)", async (reprobe) => { + const strict = { provider: "strict", model: "deepseek-v4-flash" }; + const backup = { provider: "xai", model: "grok-4.5" }; + const config = comboConfig([strict, backup]); + config.providers.strict = { + adapter: "openai-responses", baseUrl: "https://strict.example.test/v1", apiKey: "synthetic-key", + preserveResponsesReasoningContent: true, requiresAdjacentResponsesToolResults: true, + }; + config.combos!.routed!.cooldownReprobe = { enabled: reprobe }; + if (reprobe) { + for (const target of [strict, backup]) coolComboTarget("routed", target, { cooldownMs: 60_000 }); + } + const urls: string[] = []; + globalThis.fetch = (async (input) => { + const url = String(input); + urls.push(url); + return url.includes("chatgpt.com") + ? new Response(recoverySse("recovered assignment"), { headers: { "content-type": "text/event-stream" } }) + : providerCompletion(); + }) as typeof fetch; + const headers = codexHeaders(); + headers.set("session_id", `encrypted-replay-${reprobe}`); + const seeded = await post(config, "xai/grok-4.5", [{ role: "user", content: "seed identity" }], headers); + expect(seeded.status).toBe(200); + await seeded.text(); + urls.length = 0; + const response = await post(config, "combo/routed", [ + { type: "reasoning", id: "rs_foreign", summary: [], encrypted_content: "foreign-blob", content: [] }, + { type: "function_call", call_id: "call_1", name: "get_weather", arguments: "{}" }, + { type: "function_call_output", call_id: "call_1", output: "rain" }, + ...encryptedInput(), + ], headers, undefined, { tools: [{ type: "function", name: "get_weather", parameters: { type: "object", properties: {} } }] }); + await response.text(); + expect(response.status).toBe(200); + expect(urls).toHaveLength(2); + expect(urls[0]).toContain("chatgpt.com"); + expect(urls[1]).not.toContain("strict.example.test"); + if (reprobe) expect(isComboTargetInCooldown("routed", strict)).toBe(true); + }); + test("rejects an all-disabled combo before recovery creates or caches plaintext", async () => { const assignment = "MUST-NOT-BE-PRODUCED-OR-CACHED"; const config = comboConfig([{ provider: "xai", model: "grok-4.5" }]); diff --git a/tests/server/combo-metered-consent.test.ts b/tests/server/combo-metered-consent.test.ts index afa54cc8633..4cc85ad7bb2 100644 --- a/tests/server/combo-metered-consent.test.ts +++ b/tests/server/combo-metered-consent.test.ts @@ -125,7 +125,7 @@ test("a subagent fallback that collapses a combo onto its metered target needs k const config = (consent: boolean): OcxConfig => ({ port: 16389, defaultProvider: "primary", providers: { primary: provider("primary"), subscription: provider("subscription"), paid: provider("paid") }, - combos: { chain: { alias: "chain-model", targets: [{ provider: "subscription", model: "m" }, { provider: "paid", model: "m", metered: true }] } }, + combos: { chain: { alias: "chain-model", cooldownReprobe: { enabled: true }, targets: [{ provider: "subscription", model: "m" }, { provider: "paid", model: "m", metered: true }] } }, subagentModelFallback: ["chain-model"], apiKeys: [{ ...key, allowMeteredComboTargets: consent }], }); From 8c849ed335e232a6d2d674925a0848c4bb536fa5 Mon Sep 17 00:00:00 2001 From: Artur Safonov Date: Sun, 25 Oct 2026 17:54:08 +0200 Subject: [PATCH 5/5] fix(combos): admit encrypted recovery before cooldown reprobe --- src/server/responses/core-combo.ts | 17 ++++----- structure/runtime.md | 2 +- .../server/agent-task-recovery-combo.test.ts | 35 +++++++++++++++++++ 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 6badc29d8bd..64e296ade5f 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -530,16 +530,13 @@ export async function executeComboResponses( let encryptedTaskRecoveryAttempted = false; let recoveryFailureReason: AgentTaskRecoveryFailureReason | undefined; let storedPool401ReplayDispatched = false; + const recovery = agentTaskRecoveryConfig(config); + const encryptedTaskRecoveryAdmitted = (options.inboundWire ?? "responses") === "responses" + && isThreadSpawnRequest(req.headers) && !!recovery && !options.comboAttempt; const recoverUnreadableEncryptedTask = async (): Promise => { if (encryptedTaskRecoveryAttempted) return false; encryptedTaskRecoveryAttempted = true; - const recovery = agentTaskRecoveryConfig(config); - if ( - (options.inboundWire ?? "responses") !== "responses" - || !isThreadSpawnRequest(req.headers) - || !recovery - || options.comboAttempt - ) { + if (!encryptedTaskRecoveryAdmitted || !recovery) { discardEncryptedAgentTaskRecovery( req, (body as { input?: unknown } | undefined)?.input, @@ -616,7 +613,11 @@ export async function executeComboResponses( : jevSessionPinScope({ combo, comboId, admission: options.admission, lane: sessionLaneIdFromRequest(req.headers) }); if (unreadableEncryptedAgentTask && !pick) { - pick = await pickWithWait({ eligible: recoveryEligible, now: initialNow }); + pick = await pickWithWait({ + eligible: recoveryEligible, + allowCooldownReprobe: encryptedTaskRecoveryAdmitted, + now: initialNow, + }); if (!pick) { discardEncryptedAgentTaskRecovery( req, diff --git a/structure/runtime.md b/structure/runtime.md index 56d49a3a697..bc6383fb32c 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. Every combo, routing-profile, subagent and hard-lock hop to another model or provider first passes the [fallback policy](fallback-policy.md#sites). -`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. Opt-in `cooldownReprobe: { enabled: true, intervalMs }` in `src/combos/resolve.ts` permits an initial non-preview request to clear all-cooled eligible targets once per configured interval (default 300000 ms, range 1000..86400000). The synchronous reservation lives in selection state and resets on proxy restart or selection-state reset; Only target keys eligible for the current request are cleared; other target cooldowns are preserved. Route-level selection without request eligibility disables reprobes, as does the in-flight retry-after-cooldown path, which waits for normal expiry. Nested wait-path selections cannot start a second reprobe. Preparation first checks available siblings without evaluating cooled targets; a throwing eligibility predicate is ineligible for recovery. Encrypted-task recovery relaxes only payload readability and retains metering, JEV, reasoning-replay and protocol-lane eligibility. New failures record cooldowns normally, and provider quota, account admission, payload eligibility, metered-spend and cancellation checks remain authoritative. There is no timer; existing send bounds remain. The opt-in may retry a target before its server Retry-After. CLI `--cooldown-reprobe on|off` and `--cooldown-reprobe-interval` control it; omitted management fields preserve it on unrelated saves. Tests: `tests/codex-integration/combo-cooldown-reprobe.test.ts` and `tests/cli/cli-combo-parity.test.ts`. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. +`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. Opt-in `cooldownReprobe: { enabled: true, intervalMs }` in `src/combos/resolve.ts` permits an initial non-preview request to clear all-cooled eligible targets once per configured interval (default 300000 ms, range 1000..86400000). The synchronous reservation lives in selection state and resets on proxy restart or selection-state reset; Only target keys eligible for the current request are cleared; other target cooldowns are preserved. Route-level selection without request eligibility disables reprobes, as does the in-flight retry-after-cooldown path, which waits for normal expiry. Nested wait-path selections cannot start a second reprobe. Preparation first checks available siblings without evaluating cooled targets; a throwing eligibility predicate is ineligible for recovery. Encrypted-task recovery relaxes only payload readability and retains metering, JEV, reasoning-replay and protocol-lane eligibility. Its recovery selection may reserve a reprobe only after Responses ingress, thread-spawn, enabled recovery configuration and non-child-attempt admission pass; refused recovery preserves cooldowns and the reservation. New failures record cooldowns normally, and provider quota, account admission, payload eligibility, metered-spend and cancellation checks remain authoritative. There is no timer; existing send bounds remain. The opt-in may retry a target before its server Retry-After. CLI `--cooldown-reprobe on|off` and `--cooldown-reprobe-interval` control it; omitted management fields preserve it on unrelated saves. Tests: `tests/codex-integration/combo-cooldown-reprobe.test.ts` and `tests/cli/cli-combo-parity.test.ts`. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. ## Combo default effort precedence diff --git a/tests/server/agent-task-recovery-combo.test.ts b/tests/server/agent-task-recovery-combo.test.ts index 4cf057ea579..828fd6e82cb 100644 --- a/tests/server/agent-task-recovery-combo.test.ts +++ b/tests/server/agent-task-recovery-combo.test.ts @@ -11,6 +11,7 @@ import { import { resetAgentTaskRecoveryState } from "../../src/server/responses/agent-task-recovery"; import { agentTaskRecoveryCacheSnapshotForTests } from "../../src/server/responses/agent-task-recovery-cache"; import { clearComboTargetCooldowns, coolComboTarget, isComboTargetInCooldown } from "../../src/combos/failover"; +import { clearComboSelectionState } from "../../src/combos/resolve"; import { clearCachedProviderQuotas, setCachedProviderQuotaForTests, @@ -70,6 +71,7 @@ describe("combo path encrypted agent task recovery", () => { resetAgentTaskRecoveryState(); clearCachedProviderQuotas(); clearComboTargetCooldowns(); + clearComboSelectionState(); }); afterEach(() => { @@ -77,6 +79,7 @@ describe("combo path encrypted agent task recovery", () => { resetAgentTaskRecoveryState(); clearCachedProviderQuotas(); clearComboTargetCooldowns(); + clearComboSelectionState(); clearResponseStateForTests(); // Released after the state flush and before the directory holding it is removed. releaseSpendHome?.(); @@ -167,6 +170,38 @@ describe("combo path encrypted agent task recovery", () => { if (reprobe) expect(isComboTargetInCooldown("routed", strict)).toBe(true); }); + test.each(["non-spawn", "disabled-recovery"] as const)("refused %s recovery preserves cooldowns and the reprobe reservation", async (reason) => { + const target = { provider: "xai", model: "grok-4.5" }; + const config = comboConfig([target]); + config.combos!.routed!.cooldownReprobe = { enabled: true, intervalMs: 86400000 }; + coolComboTarget("routed", target, { cooldownMs: 60_000 }); + const headers = codexHeaders(); + if (reason === "non-spawn") headers.delete("x-openai-subagent"); + else config.agentTaskRecovery = { enabled: false }; + const urls: string[] = []; + globalThis.fetch = (async (input) => { + const url = String(input); + urls.push(url); + return url.includes("chatgpt.com") + ? new Response(recoverySse("admitted assignment"), { headers: { "content-type": "text/event-stream" } }) + : providerCompletion(); + }) as typeof fetch; + const refused = await post(config, "combo/routed", encryptedInput(), headers); + await refused.text(); + expect(refused.status).toBe(503); + expect(urls).toEqual([]); + expect(isComboTargetInCooldown("routed", target)).toBe(true); + expect(agentTaskRecoveryCacheSnapshotForTests()).toEqual({ entries: 0, bytes: 0 }); + + // An admitted request immediately afterwards must still own the first recovery pass. + config.agentTaskRecovery = { enabled: true }; + const admitted = await post(config, "combo/routed", encryptedInput(), codexHeaders()); + await admitted.text(); + expect(admitted.status).toBe(200); + expect(urls).toHaveLength(2); + expect(isComboTargetInCooldown("routed", target)).toBe(false); + }); + test("rejects an all-disabled combo before recovery creates or caches plaintext", async () => { const assignment = "MUST-NOT-BE-PRODUCED-OR-CACHED"; const config = comboConfig([{ provider: "xai", model: "grok-4.5" }]);