feat(combos): add opt-in cooldown recovery with configurable interval

docs-site/src/content/docs/guides/combos.md

@@ -1029,3 +1029,31 @@ validation message.
 
 The error was terminal rather than target-specific. Fix invalid input, reduce an oversized context,
 handle a policy refusal, or correct the rejected request origin. Combos do not hop for those cases.
+
+## Retry all-cooled targets (opt-in)
+
+Combo cooldown recovery is opt-in and disabled by default. To allow one new
+failover pass when every otherwise eligible target has a saved combo cooldown:
+
+```sh
+ocx combo set review --cooldown-reprobe on --cooldown-reprobe-interval 300000
+ocx combo set review --cooldown-reprobe off
+```
+
+The corresponding per-combo configuration is:
+
+```json
+{ "cooldownReprobe": { "enabled": true, "intervalMs": 300000 } }
+```
+
+The default interval is five minutes; accepted values are integer milliseconds
+from 1000 to 86400000. Setting the interval alone does not enable recovery.
+Recovery is demand-driven: the first eligible request can reserve a pass immediately,
+and later passes are separated by at least the configured interval. There is no timer
+or background traffic. Each target can be attempted once in the request's existing
+failover budget; failures record fresh cooldowns. Account quota checks, disabled
+providers, request compatibility, metered-spend permissions and cancellation remain
+enforced. Explicit provider Retry-After values may therefore be retried early only
+when the operator enables this option. Preview/list operations never clear cooldowns.
+The interval is tracked per combo in memory and resets on proxy restart or combo
+selection-state reset. Existing cooldown-wait behavior is unchanged when disabled.

scripts/test-layout/layout-combos.json

@@ -0,0 +1,3 @@
+{
+  "combo-cooldown-reprobe.test.ts": "codex-integration"
+}

scripts/test-layout/layout.json

@@ -1,7 +1,7 @@
 {
   "version": 1,
   "root": "tests",
-  "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json"],
+  "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json", "layout-combos.json"],
   "explicit": { "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "codex-provider-routing.test.ts": "codex-integration", "config-native-reasoning-retention.test.ts": "config", "server-native-reasoning-retention-merge.test.ts": "server", "server-native-reasoning-retention.test.ts": "server", "responses-retained-reasoning-stream-recovery.test.ts": "responses", "openai-native-hosted-search-history.test.ts": "adapters/openai", "responses-native-compact-search-history.test.ts": "responses", "config-hosted-search-projection.test.ts": "config", "gcp-credential-boundary.test.ts": "adapters/google", "gcp-sa-marker.test.ts": "adapters/google", "gcp-sa-adapter.test.ts": "adapters/google", "management-gcp-credential-input.test.ts": "server", "codex-preserve-signed-in.test.ts": "codex-integration", "cli-account-preservation.test.ts": "cli", "cli-provider-list-active.test.ts": "cli", "service-state-isolation.test.ts": "service", "compaction-history-framing.test.ts": "responses", "catalog-native-max-context.test.ts": "codex-integration", "protocol-argon-thought-boundary.test.ts": "responses", "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "bridge-terminal-refusal.test.ts": "adapters", "loop-terminal-refusal.test.ts": "images", "web-search-terminal-refusal.test.ts": "web-search", "combo-terminal-refusal.test.ts": "routing", "server-terminal-refusal-wiring.test.ts": "server", "incomplete-reason-normalization.test.ts": "responses", "advisor-retention.test.ts": "advisor", "responses-native-key-cooldown.test.ts": "responses", "responses-run-turn-key-readmission.test.ts": "responses", "qoder-spawn-readmission.test.ts": "adapters", "api-key-selection-cooldown.test.ts": "providers", "api-key-auto-pause-distribution.test.ts": "providers", "explicit-key-pause-oauth-boundary.test.ts": "providers", "server-chat-native-key-selection.test.ts": "server", "compact-key-selection.test.ts": "responses", "messages-native-key-selection.test.ts": "claude-integration", "coding-agent-windows-dispatch.test.ts": "adapters", "coding-agent-native-empty-argv.test.ts": "windows",
     "service-proxy-env.test.ts": "service", "openai-chat-standalone-bare-block.test.ts": "adapters/openai",
     "owner-registry-isolation.test.ts": "config",

skills/ocx/references/01_surface_agents-routing.md

@@ -958,7 +958,7 @@ State-changing: no. JSON mode: `payload`.
 
 ### `ocx combo set`
 
-Usage: `ocx combo set <id> [--targets <provider/model[:weight],...> | --targets-file <FILE|->] [--strategy <failover|round-robin|random|least-used|reset-window|jev>] [--sticky <1-100|->] [--effort <low|medium|high|xhigh|max|ultra|->] [--effort-mode <fallback|force|->] [--alias <name|->] [--native-alias [on|off]] [--allow-metered-targets <on|off>] [--display-name <label|->] [--description <text|->] [--decision-provider <provider|-> | --decision-model <route|->] [--decision-timeout <ms|->] [--decision-quota-signals <on|off|->] [--decision-quota-tiers <JSON|->] [--session-pin <duration|->] [--rename-from <id>] [--image-input <auto|disabled>] [--reasoning-effort-mode <strict|adaptive>] [--json]`
+Usage: `ocx combo set <id> [--targets <provider/model[:weight],...> | --targets-file <FILE|->] [--strategy <failover|round-robin|random|least-used|reset-window|jev>] [--sticky <1-100|->] [--cooldown-reprobe <on|off>] [--cooldown-reprobe-interval <ms>] [--effort <low|medium|high|xhigh|max|ultra|->] [--effort-mode <fallback|force|->] [--alias <name|->] [--native-alias [on|off]] [--allow-metered-targets <on|off>] [--display-name <label|->] [--description <text|->] [--decision-provider <provider|-> | --decision-model <route|->] [--decision-timeout <ms|->] [--decision-quota-signals <on|off|->] [--decision-quota-tiers <JSON|->] [--session-pin <duration|->] [--rename-from <id>] [--image-input <auto|disabled>] [--reasoning-effort-mode <strict|adaptive>] [--json]`
 
 Create, update or rename a combo.
 
@@ -975,6 +975,8 @@ State-changing: yes. JSON mode: `payload`.
 | `--targets` | string | Ordered provider/model[:weight] list; required for creation. |
 | `--strategy` | string | Routing strategy. |
 | `--sticky` | string | Round-robin limit 1-100; - resets. |
+| `--cooldown-reprobe` | string | Opt into one recovery pass when all eligible targets cool; off by default. |
+| `--cooldown-reprobe-interval` | string | Minimum interval between recovery passes in ms (1000..86400000); default 300000. |
 | `--effort` | string | Default effort; - clears. |
 | `--effort-mode` | string | fallback or force; - resets to fallback. |
 | `--alias` | string | Alias; - clears. |

src/cli/capabilities-agents-routing.ts

@@ -385,10 +385,10 @@ export const AGENT_ROUTING_CAPABILITIES: readonly Capability[] = [
   },
   {
     command: ["combo","set"],
-    usage: "ocx combo set <id> [--targets <provider/model[:weight],...> | --targets-file <FILE|->] [--strategy <failover|round-robin|random|least-used|reset-window|jev>] [--sticky <1-100|->] [--effort <low|medium|high|xhigh|max|ultra|->] [--effort-mode <fallback|force|->] [--alias <name|->] [--native-alias [on|off]] [--allow-metered-targets <on|off>] [--display-name <label|->] [--description <text|->] [--decision-provider <provider|-> | --decision-model <route|->] [--decision-timeout <ms|->] [--decision-quota-signals <on|off|->] [--decision-quota-tiers <JSON|->] [--session-pin <duration|->] [--rename-from <id>] [--image-input <auto|disabled>] [--reasoning-effort-mode <strict|adaptive>] [--json]",
+    usage: "ocx combo set <id> [--targets <provider/model[:weight],...> | --targets-file <FILE|->] [--strategy <failover|round-robin|random|least-used|reset-window|jev>] [--sticky <1-100|->] [--cooldown-reprobe <on|off>] [--cooldown-reprobe-interval <ms>] [--effort <low|medium|high|xhigh|max|ultra|->] [--effort-mode <fallback|force|->] [--alias <name|->] [--native-alias [on|off]] [--allow-metered-targets <on|off>] [--display-name <label|->] [--description <text|->] [--decision-provider <provider|-> | --decision-model <route|->] [--decision-timeout <ms|->] [--decision-quota-signals <on|off|->] [--decision-quota-tiers <JSON|->] [--session-pin <duration|->] [--rename-from <id>] [--image-input <auto|disabled>] [--reasoning-effort-mode <strict|adaptive>] [--json]",
     summary: "Create, update or rename a combo.",
     routes: [{"method":"GET","path":"/api/combos"},{"method":"PUT","path":"/api/combos"}],
-    flags: [{"name":"--allow-metered-targets","value":"string","summary":"on grants operator consent to metered targets only on a loopback-only proxy; off clears it."},{"name":"--targets","value":"string","summary":"Ordered provider/model[:weight] list; required for creation."},{"name":"--strategy","value":"string","summary":"Routing strategy."},{"name":"--sticky","value":"string","summary":"Round-robin limit 1-100; - resets."},{"name":"--effort","value":"string","summary":"Default effort; - clears."},{"name":"--effort-mode","value":"string","summary":"fallback or force; - resets to fallback."},{"name":"--alias","value":"string","summary":"Alias; - clears."},{"name":"--display-name","value":"string","summary":"Label; - clears."},{"name":"--description","value":"string","summary":"Catalog row description; - clears."},{"name":"--decision-provider","value":"string","summary":"JEV provider; - clears."},{"name":"--decision-model","value":"string","summary":"JEV route; - clears; exclusive with provider."},{"name":"--decision-timeout","value":"string","summary":"JEV deadline 1000-120000 ms; - clears."},{"name":"--decision-quota-signals","value":"string","summary":"JEV advisory quota on/off; - clears."},{"name":"--decision-quota-tiers","value":"string","summary":"Partial JSON thresholds; - clears."},{"name":"--session-pin","value":"string","summary":"JEV session pin idle window 1m-24h (90s, 20m, 1h); - clears."},{"name":"--rename-from","value":"string","summary":"Existing combo ID to rename."},{"name":"--json","value":"boolean","summary":"Emit structured JSON."},{"name":"--native-alias","value":"string","summary":"Bare legacy flag means on; explicit on/off preserves presence, including false."},{"name":"--targets-file","value":"string","summary":"Bounded ordered target JSON array, exclusive with --targets; preserves effort/profile/last-resort metadata."},{"name":"--image-input","value":"string","summary":"auto or disabled; explicit auto clears the previous disabled choice."},{"name":"--reasoning-effort-mode","value":"string","summary":"strict or adaptive, distinct from --effort-mode fallback/force."}],
+    flags: [{"name":"--allow-metered-targets","value":"string","summary":"on grants operator consent to metered targets only on a loopback-only proxy; off clears it."},{"name":"--targets","value":"string","summary":"Ordered provider/model[:weight] list; required for creation."},{"name":"--strategy","value":"string","summary":"Routing strategy."},{"name":"--sticky","value":"string","summary":"Round-robin limit 1-100; - resets."},{"name":"--cooldown-reprobe","value":"string","summary":"Opt into one recovery pass when all eligible targets cool; off by default."},{"name":"--cooldown-reprobe-interval","value":"string","summary":"Minimum interval between recovery passes in ms (1000..86400000); default 300000."},{"name":"--effort","value":"string","summary":"Default effort; - clears."},{"name":"--effort-mode","value":"string","summary":"fallback or force; - resets to fallback."},{"name":"--alias","value":"string","summary":"Alias; - clears."},{"name":"--display-name","value":"string","summary":"Label; - clears."},{"name":"--description","value":"string","summary":"Catalog row description; - clears."},{"name":"--decision-provider","value":"string","summary":"JEV provider; - clears."},{"name":"--decision-model","value":"string","summary":"JEV route; - clears; exclusive with provider."},{"name":"--decision-timeout","value":"string","summary":"JEV deadline 1000-120000 ms; - clears."},{"name":"--decision-quota-signals","value":"string","summary":"JEV advisory quota on/off; - clears."},{"name":"--decision-quota-tiers","value":"string","summary":"Partial JSON thresholds; - clears."},{"name":"--session-pin","value":"string","summary":"JEV session pin idle window 1m-24h (90s, 20m, 1h); - clears."},{"name":"--rename-from","value":"string","summary":"Existing combo ID to rename."},{"name":"--json","value":"boolean","summary":"Emit structured JSON."},{"name":"--native-alias","value":"string","summary":"Bare legacy flag means on; explicit on/off preserves presence, including false."},{"name":"--targets-file","value":"string","summary":"Bounded ordered target JSON array, exclusive with --targets; preserves effort/profile/last-resort metadata."},{"name":"--image-input","value":"string","summary":"auto or disabled; explicit auto clears the previous disabled choice."},{"name":"--reasoning-effort-mode","value":"string","summary":"strict or adaptive, distinct from --effort-mode fallback/force."}],
     mutates: true,
     json: "payload",
     details: ["create and update are aliases of set. Existing fields are read before writing; --targets replaces targets. Without --strategy an existing combo keeps its stored strategy and JEV policy; a new combo defaults to failover.","force overrides valid client effort and may increase cost and latency.","Only the implemented target grammar is supported; full GUI target metadata and routing-profile writes are separate tasks.","New file/flag conflicts refuse before transport. Partial edits preserve target metadata; explicit false/default values override server carry. Turning a native alias off may also require clearing its incompatible alias.","Read/merge/write uses one target and remains an upsert without revision CAS. Saved-but-incomplete catalog outcomes return nonzero with their receipt."],

src/cli/combo-result.ts

@@ -40,6 +40,13 @@ function publicCombo(value: unknown): Record<string, unknown> {
       if (!tiers) invalid();
       result[key] = tiers;
       continue;
+    } else if (key === "cooldownReprobe") {
+      if (!record(child) || typeof child.enabled !== "boolean"
+        || Object.keys(child).some(field => field !== "enabled" && field !== "intervalMs")) invalid();
+      if (child.intervalMs !== undefined && (typeof child.intervalMs !== "number"
+        || !Number.isSafeInteger(child.intervalMs) || child.intervalMs < 1000 || child.intervalMs > 86_400_000)) invalid();
+      result[key] = { enabled: child.enabled, ...(child.intervalMs !== undefined ? { intervalMs: child.intervalMs } : {}) };
+      continue;
     } else if (key === "sessionPin") {
       const pin = normalizeJevSessionPin(child);
       if (!pin) invalid();

src/cli/combo.ts

@@ -24,6 +24,7 @@ const USAGE = `Usage:
   ocx combo show <id> [--json]
   ocx combo set <id> [--targets <provider/model[:weight],...> | --targets-file <FILE|->]
       [--strategy <failover|round-robin|random|least-used|reset-window|jev>] [--sticky <1-100|->]
+      [--cooldown-reprobe <on|off>] [--cooldown-reprobe-interval <ms>]
       [--effort <low|medium|high|xhigh|max|ultra|->] [--effort-mode <fallback|force|->]
       (force overrides valid client effort and can increase cost/latency) [--alias <name|->]
       [--native-alias [on|off]] [--allow-metered-targets <on|off>] [--display-name <label|->]
@@ -114,6 +115,16 @@ async function set(argv: string[], deps: RuntimeApiDeps): Promise<number> {
     throw new CliUsageError("--sticky must be an integer from 1 to 100, or -", USAGE);
   }
   if (stickyLimit !== undefined && stickyLimit > 100) throw new CliUsageError("--sticky must be <= 100", USAGE);
+  const reprobe = takeOption(args, "--cooldown-reprobe");
+  const reprobeInterval = takeOption(args, "--cooldown-reprobe-interval");
+  if (reprobe !== undefined && reprobe !== "on" && reprobe !== "off") {
+    throw new CliUsageError("--cooldown-reprobe must be on or off", USAGE);
+  }
+  const reprobeIntervalMs = reprobeInterval === undefined ? undefined : Number(reprobeInterval);
+  if (reprobeIntervalMs !== undefined && (!Number.isSafeInteger(reprobeIntervalMs)
+    || reprobeIntervalMs < 1000 || reprobeIntervalMs > 86_400_000)) {
+    throw new CliUsageError("--cooldown-reprobe-interval must be an integer from 1000 to 86400000 ms", USAGE);
+  }
   const effort = takeOption(args, "--effort");
   const effortMode = takeOption(args, "--effort-mode");
   if (effortMode !== undefined && effortMode !== "fallback" && effortMode !== "force" && effortMode !== "-") {
@@ -180,6 +191,13 @@ async function set(argv: string[], deps: RuntimeApiDeps): Promise<number> {
   const combo: Record<string, unknown> = existing
     ? Object.fromEntries(Object.entries(existing).filter(([, value]) => value !== null))
     : { strategy, stickyLimit: stickyLimit ?? 1 };
+  if (reprobe !== undefined || reprobeIntervalMs !== undefined) {
+    const previous = (combo.cooldownReprobe ?? {}) as { enabled?: boolean; intervalMs?: number };
+    combo.cooldownReprobe = { ...previous,
+      enabled: reprobe === undefined ? previous.enabled === true : reprobe === "on",
+      ...(reprobeIntervalMs !== undefined ? { intervalMs: reprobeIntervalMs } : {}),
+    };
+  }
   delete combo.id;
   delete combo.model;
   if (replacingTargets) combo.targets = additional.targets ?? parsedTargets;

src/combos/failover.ts

@@ -313,14 +313,19 @@ export function sweepExpiredComboTargetCooldowns(now = Date.now()): number {
   return removed;
 }
 
-export function clearComboTargetCooldowns(comboId?: string): void {
+/** Clear all cooldowns, one combo, or only the supplied target keys within that combo. */
+export function clearComboTargetCooldowns(comboId?: string, targetKeys?: Iterable<string>): void {
   if (comboId === undefined) {
     targetCooldowns.clear();
     liveComboTargets.clear();
     lastReconciledGeneration = 0;
     return;
   }
   const prefix = `${comboId}\0`;
+  if (targetKeys !== undefined) {
+    for (const key of targetKeys) targetCooldowns.delete(`${prefix}${key}`);
+    return;
+  }
   for (const key of targetCooldowns.keys()) {
     if (key.startsWith(prefix)) targetCooldowns.delete(key);
   }