diff --git a/docs-site/src/content/docs/reference/proxy-formats.md b/docs-site/src/content/docs/reference/proxy-formats.md index 6fd9561221d..e2e81458451 100644 --- a/docs-site/src/content/docs/reference/proxy-formats.md +++ b/docs-site/src/content/docs/reference/proxy-formats.md @@ -688,3 +688,17 @@ that repair, it becomes a normal user message. If a current v2 task remains genu but the selected routed target cannot read native ChatGPT ciphertext, opencodex fails with `unreadable_encrypted_agent_task` instead of sending unreadable bytes to that provider. See [Sub-agent Surface](/guides/sub-agent-surface/) for the client behavior around worker tasks. + +History is handled too, and differently, because losing a replayed message should not end a +conversation. A replayed `agent_message` that mixes readable text with backend ciphertext cannot +be lowered to a public message, so a routed Responses destination would otherwise receive the +ciphertext along with an item type only the ChatGPT backend declares. Before dispatch, opencodex +replaces that ciphertext with `[encrypted content omitted]` — the same marker it already +substitutes after an upstream decrypt failure — which leaves the item lowerable and the readable +text intact. The provider never sees the ciphertext or the private item, and the conversation +continues. Combo targets are repaired individually, since each receives its own copy of the +request. The canonical ChatGPT Codex backend is exempt because it is the destination that minted +and can read those bytes; a `forward` provider pointed at any other origin is not exempt. +Explicitly trusted `allowEncryptedV2AgentTasks` routes and translated Chat or Anthropic wires are +unaffected, as are other item types such as reasoning and tool-output blobs, which keep their +existing decrypt-failure recovery. diff --git a/src/server/responses.ts b/src/server/responses.ts index 2a446619a26..4187e4ff2e3 100644 --- a/src/server/responses.ts +++ b/src/server/responses.ts @@ -5,7 +5,7 @@ import { requestPacingOverloadResponse } from "./responses/pacing-overload"; export { buildToolBridgeMaps, isV1CollabSurface, collabSurface, multiAgentGuidanceText, V2_GUIDANCE_CHAR_BUDGET, injectDeveloperMessage } from "./responses/collaboration"; export type { MultiAgentGuidanceOptions, MultiAgentGuidanceDeps } from "./responses/collaboration"; -export { hasUnreadableEncryptedAgentTask, sanitizeEncryptedContentInPlace } from "./responses/encrypted-payload"; +export { hasUnreadableEncryptedAgentTask, sanitizeEncryptedContentInPlace, stripAgentMessageCiphertextInPlace } from "./responses/encrypted-payload"; export { COMPACT_RESPONSE_MAX_BYTES, bufferCompactResponse } from "./responses/compact"; export { disableResponsesRequestTimeout, safeHostLabel, fetchWithHeaderTimeout } from "./responses/fetch-helpers"; export { sidecarOutcomeRecorder, isShadowSourceModel, codexLogAccountId, usesCodexForwardPoolAuth, codexForwardTerminalOutcomeRecorder, decodeRequestErrorResponse, buildComboChildHeaders, linkAbortSignal } from "./responses/core"; diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 87124ab57f0..bfd3a5dc7eb 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -394,7 +394,7 @@ import type { EffectiveSubagentRoster, SpawnAgentSurface } from "../../codex/cat import { buildToolBridgeMaps, collabSurface, injectDeveloperMessage, multiAgentGuidanceText } from "./collaboration"; import { mapCodexAuthContextErrorToResponse, nativeMainRefreshFailureResponse } from "./codex-auth-error"; -import { hasUnreadableEncryptedAgentTask, looksLikeBackendCiphertext, sanitizeEncryptedContentInPlace } from "./encrypted-payload"; +import { hasUnreadableEncryptedAgentTask, looksLikeBackendCiphertext, sanitizeEncryptedContentInPlace, stripAgentMessageCiphertextInPlace } from "./encrypted-payload"; import { fetchWithHeaderTimeout, providerFetch, safeHostLabel, safeOriginLabel, storedPoolReplayDispatchNotifier, type ProviderFetchOptions } from "./fetch-helpers"; import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability"; import { @@ -4024,6 +4024,49 @@ async function handleResponsesInner( return unreadableEncryptedAgentTaskResponse(recoveryFailureReason); } + // The guard above asks whether the CURRENT worker task is readable, and it only inspects the + // tail item. An `agent_message` that mixes readable text with backend ciphertext answers + // "readable" to that question at every position, so it passed -- and then + // `normalizeRoutedAgentMessages` refused to lower it, because lowering requires every part to + // be representable. The raw Responses passthrough serialized the private item as it stood, so + // backend ciphertext and an item type only the Codex backend declares reached a third-party + // provider, which answered `422 unknown item type "agent_message"` (#4454). + // + // The opaque-blob path already knows the repair: replace the undecryptable part with an + // omission marker, which leaves the item lowerable. It applied that repair only AFTER an + // upstream rejection. For a destination that cannot accept the private item under any + // circumstances, that round trip was never going to succeed and sent the ciphertext to find + // out, so do the repair here instead. Recovery above has already had its chance to turn the + // same bytes into real plaintext; only what it could not rescue reaches this. + if (inboundWire === "responses" && !finalRouteCanPassThroughEncryptedTask) { + // Only the raw Responses passthrough puts input items on the wire verbatim, so that is the + // only wire this has to repair: translated wires rebuild the body from parsed messages, where + // `inputContentParts` drops an encrypted part instead of forwarding it. The exemption is the + // canonical Codex backend alone, because it is the one destination that minted these bytes and + // can read them. `authMode: "forward"` is NOT that test -- a noncanonical forward gateway is + // somebody else's server that happens to be configured for passthrough, and it receives the + // ciphertext like any other third party. + // + // Combo children run this too. Each child carries its own `structuredClone` of the body + // (`concreteComboRequestBody`) and its own concrete route, so a sibling's repair is invisible + // here and a target that resolves to a routed Responses wire would otherwise send the + // ciphertext that the parent's own dispatch no longer does. + const wireProvider = resolveWireProtocolOverride( + route.providerName, + route.modelId, + route.provider, + inboundWire, + ); + if (wireProvider.adapter === "openai-responses" && !isCanonicalOpenAiForwardProvider(wireProvider)) { + const repaired = stripAgentMessageCiphertextInPlace((body as { input?: unknown } | undefined)?.input); + if (repaired > 0) { + console.warn( + `[opencodex] replaced ciphertext in ${repaired} replayed agent message(s) with an omission marker; the selected provider cannot read native ChatGPT ciphertext`, + ); + } + } + } + // The canonical ChatGPT backend rejects previous_response_id, so a local replay miss leaves no // safe way to recover the omitted history. Fail before auth, adapter construction, or upstream // I/O instead of stripping the id and silently forwarding a context-free delta (#702). diff --git a/src/server/responses/encrypted-payload.ts b/src/server/responses/encrypted-payload.ts index 82d4b0514eb..066bb9b5226 100644 --- a/src/server/responses/encrypted-payload.ts +++ b/src/server/responses/encrypted-payload.ts @@ -319,6 +319,167 @@ export function hasEncryptedContentPart(content: unknown): boolean { } +/** The marker `prepareOpaqueBlobRecovery` already substitutes for an undecryptable part. */ +export const OMITTED_ENCRYPTED_CONTENT_TEXT = "[encrypted content omitted]"; + +/** + * The Fernet WIRE shape, without validating the body: version prefix, base64url alphabet, and a + * canonical encoded length. Free text is judged by this rather than by + * `looksLikeBackendCiphertext`, which is length >= 64 over a character class that a SHA-256 hex + * digest matches exactly at 64 characters -- as do a SHA-512 digest, a long key, and adjacent + * short encoded fragments. An `encrypted_content` slot carries ciphertext by definition and is + * stripped whatever it holds; a text part does not, and replacing a digest a child deliberately + * printed would destroy readable content to protect bytes that were never secret. + */ +const FERNET_SHAPED = /^g[A-Za-z0-9_-]+={0,2}$/; + +function looksLikeFernetToken(text: string): boolean { + return text.length >= 100 && text.length % 4 === 0 && FERNET_SHAPED.test(text); +} + +function textWithRunsOmitted(payload: string, runs: readonly FernetTokenRun[]): string { + let last = 0; + let out = ""; + for (const run of runs) { + out += payload.slice(last, run.index) + OMITTED_ENCRYPTED_CONTENT_TEXT; + last = run.index + run.token.length; + } + return out + payload.slice(last); +} + +/** + * Replace ciphertext inside `agent_message` items with an omission marker, so + * `normalizeRoutedAgentMessages` can lower them onto public messages. Returns how many items + * were repaired. Items are replaced rather than mutated, and every other item type is left + * alone: reasoning and function-output blobs keep their own reactive recovery. + * + * `hasUnreadableEncryptedAgentTask` above answers a different question: can the CURRENT worker + * task be read at all? It inspects only the tail item and reports false the moment any plaintext + * survives the envelope. `normalizeRoutedAgentMessages` asks the opposite question -- is EVERY + * part lowerable? -- and forwards the private item verbatim when one is not. A mixed + * `input_text` + `encrypted_content` item answers "readable" to the first and "not lowerable" + * to the second, so it fell between them: the guard never fired, the adapter refused to lower it, + * and the raw Responses passthrough put a private item and backend ciphertext on the wire + * (#4454). Position was never the discriminator -- a replayed child result lands mid-history and + * the tail-only scan cannot see it -- but the tail is equally exposed when it is mixed. + * + * This is the repair `prepareOpaqueBlobRecovery` performs after an upstream rejection, applied + * before dispatch for a destination that cannot accept the private item under any circumstances. + * The round trip it replaces was never going to succeed, and it sent ciphertext to a third party + * to find that out. Nothing is decrypted, and nothing readable is lost: the parent could not read + * these bytes either. + * + * The two kinds of slot are judged differently, because they carry different guarantees. An + * `encrypted_content` slot holds ciphertext by definition, so it is stripped whatever it holds: + * demanding a well-formed token there would reopen this defect one payload later, since a + * truncated token, a standard-base64 blob carrying `+` or `/`, an unexpected version byte, or a + * run past the recovery size limits would each keep the item and forward the bytes. + * + * A text part carries no such guarantee, so it is matched strictly: embedded runs that validate + * as Fernet, or a whole slot with the Fernet wire shape. A loose character-class test would be + * worse than the defect for that half -- a SHA-256 digest is exactly 64 characters of + * `[A-Za-z0-9]` and would be replaced with a marker, silently deleting something a child + * deliberately printed. + */ +export function stripAgentMessageCiphertextInPlace(input: unknown): number { + if (!Array.isArray(input)) return 0; + let repaired = 0; + for (let index = 0; index < input.length; index += 1) { + const item = input[index]; + if (!item || typeof item !== "object" || Array.isArray(item)) continue; + const record = item as Record; + if (record.type !== "agent_message") continue; + const content = record.content; + if (typeof content === "string") { + const replaced = textWithoutCiphertext(content); + if (replaced === content) continue; + input[index] = { ...record, content: replaced }; + repaired += 1; + continue; + } + if (!Array.isArray(content)) continue; + const parts = contentWithoutCiphertext(content); + if (parts === content) continue; + input[index] = { ...record, content: parts }; + repaired += 1; + } + return repaired; +} + +/** Free text: drop embedded token runs, and replace a slot that is nothing but a token. */ +function textWithoutCiphertext(text: string): string { + const runs = fernetTokenRuns(text); + if (runs.length > 0) return textWithRunsOmitted(text, runs); + return looksLikeFernetToken(text.trim()) ? OMITTED_ENCRYPTED_CONTENT_TEXT : text; +} + +function ciphertextTextOfPart(part: unknown): string | undefined { + if (!part || typeof part !== "object") return undefined; + const record = part as { type?: unknown; text?: unknown }; + return (record.type === "input_text" || record.type === "text") && typeof record.text === "string" + ? record.text + : undefined; +} + +/** + * Adjacent text slots that are one token between them. Each fragment can be too short to judge on + * its own, which is the text-side twin of the split `encrypted_content` run. The join must still + * be Fernet-shaped, so two ordinary encoded fragments do not become a marker by being adjacent. + */ +function joinedCiphertextTextParts(content: readonly unknown[]): Set { + const flagged = new Set(); + let run: Array<{ part: object; text: string }> = []; + const finish = (): void => { + if (run.length > 1 && looksLikeFernetToken(run.map(entry => entry.text).join(""))) { + for (const entry of run) flagged.add(entry.part); + } + run = []; + }; + for (const part of content) { + const text = ciphertextTextOfPart(part); + if (text === undefined || text.trim().length === 0 || !/^[A-Za-z0-9_-]+={0,2}$/.test(text)) { + finish(); + continue; + } + run.push({ part: part as object, text }); + } + finish(); + return flagged; +} + +function contentWithoutCiphertext(content: unknown[]): unknown[] { + let changed = false; + const joined = joinedCiphertextTextParts(content); + const parts = content.map((part: unknown) => { + if (!part || typeof part !== "object") return part; + if (joined.has(part)) { + changed = true; + return { type: "input_text", text: OMITTED_ENCRYPTED_CONTENT_TEXT }; + } + const record = part as { type?: unknown; text?: unknown; encrypted_content?: unknown }; + if (record.type === "encrypted_content" && typeof record.encrypted_content === "string") { + changed = true; + // Keep whatever plaintext a recognizable slot carries around its token; a slot this + // cannot parse is replaced whole rather than forwarded on the chance that it is benign. + const runs = fernetTokenRuns(record.encrypted_content); + return { + type: "input_text", + text: runs.length > 0 + ? textWithRunsOmitted(record.encrypted_content, runs) + : OMITTED_ENCRYPTED_CONTENT_TEXT, + }; + } + const text = ciphertextTextOfPart(part); + if (text === undefined) return part; + const replaced = textWithoutCiphertext(text); + if (replaced === text) return part; + changed = true; + return { ...record, text: replaced }; + }); + return changed ? parts : content; +} + + export function sanitizeEncryptedContentInPlace(input: unknown): number { if (!Array.isArray(input)) return 0; diff --git a/structure/subagents.md b/structure/subagents.md index 98c827ffa7d..6cd17a3b91b 100644 --- a/structure/subagents.md +++ b/structure/subagents.md @@ -148,6 +148,66 @@ unreadable split-token shapes. The sanitizer preserves just those fragment objec normalizing independent plaintext slots. Detection never authorizes reconstruction or recovery; other fragment layouts and mixed readable content retain their documented residual boundaries. +## Routed agent-message ciphertext egress + +Two questions about an `agent_message` were asked in two places, and the gap between them was +open. `hasUnreadableEncryptedAgentTask` asks whether the CURRENT worker task can be read and +inspects only the tail item; `normalizeRoutedAgentMessages` asks whether EVERY part can be lowered +onto a public message and forwards the private item verbatim when one cannot. An item mixing +`input_text` with `encrypted_content` is readable by the first measure and unlowerable by the +second, so it passed the guard, kept its private type through the raw Responses passthrough, and +left the process as backend ciphertext plus an item type only the Codex backend declares. The +destination answered `422 unknown item type "agent_message"` after the bytes were already sent. +Position was incidental: a replayed child result sits mid-history, where a tail-only scan cannot +see it, and the tail is exposed the same way once it is mixed. + +The repair already existed reactively. `prepareOpaqueBlobRecovery` replaces an undecryptable part +with `[encrypted content omitted]`, which leaves the item lowerable, and it ran after an upstream +rejection. A destination that cannot accept the private item under any circumstances was never +going to answer that request, so the round trip only served to send the ciphertext. +`stripAgentMessageCiphertextInPlace` in `src/server/responses/encrypted-payload.ts` applies the +same repair before dispatch, and `src/server/responses/core.ts` runs it against the final route, +after `expandPreviousResponseInput`, after the sanitizer has rewritten plaintext parked in +encrypted slots, and after encrypted-task recovery has had its chance to produce real plaintext +instead of a marker. + +The two kinds of slot are judged differently, because they carry different guarantees. An +`encrypted_content` slot holds ciphertext by definition, so it is stripped whatever it holds: +demanding a well-formed token there would reopen the same defect one payload later, since a +truncated token, a standard-base64 blob carrying `+` or `/`, an unexpected version byte, or a run +past the recovery size limits would each keep the item and forward the bytes. A text part carries +no such guarantee, so it is matched strictly -- embedded runs that validate as Fernet, or a whole +slot with the Fernet wire shape, which is the version prefix, the base64url alphabet and a +canonical length of at least 100 divisible by four. Adjacent text fragments are joined before that +test, so a token split across slots is still caught. `looksLikeBackendCiphertext` is deliberately +NOT used on text: it is length >= 64 over a character class that a SHA-256 digest matches exactly +at 64 characters, and replacing a digest a child deliberately printed would delete readable content +to protect bytes that were never secret. Other item types are untouched: reasoning and +function-output blobs keep the reactive opaque-blob recovery, which still rescues a destination +that merely failed to decrypt something it was entitled to read, and which stays reachable for the +canonical backend and for explicitly trusted routes. + +The repair resolves the same wire override the adapter is built from rather than restating routing +policy, and runs for `openai-responses` whenever the destination is not the canonical Codex +backend. `authMode: "forward"` is deliberately not that test: it describes how this proxy treats +credentials, not who answers, and a forward-configured gateway at another origin receives the +ciphertext like any third party. Only `isCanonicalOpenAiForwardProvider` is exempt, because it +alone minted these bytes and can read them. The wire override matters for the reported destination, +where the provider row names the Chat wire and a registry model default moves the model onto +Responses. Translated wires are untouched because `inputContentParts` drops an encrypted part +instead of forwarding it, and `canPassThroughEncryptedV2AgentTask` keeps an explicitly trusted +route exempt. Combo children run the repair themselves: `concreteComboRequestBody` gives each +target its own `structuredClone` and its own concrete route, so a sibling's repair is invisible to +them and a target resolving to a routed Responses wire would otherwise send what the parent's own +dispatch no longer does. + +Nothing here decrypts, and the tail NEW_TASK envelope keeps `unreadable_encrypted_agent_task` and +its opt-in recovery unchanged: an unreadable current task still fails closed rather than reaching a +child with a marker where its assignment should be. An `agent_message` carrying unknown parts but +no ciphertext still reaches the wire unchanged and still draws the destination's own 422, which is +a compatibility gap rather than an egress one. Covered by +`tests/server/v2-agent-message-failfast.test.ts`. + ## Subagents New non-OAuth provider registrations carry `initialModelSelection` with a unique diff --git a/structure/transports/responses.md b/structure/transports/responses.md index c00d08f18c8..1f1bd3cd2fb 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -288,12 +288,12 @@ call when `isXaiResponsesDestination` recognizes HTTPS `api.x.ai` or `cli-chat-p on the standard port. A nonblank string becomes one `input_text` part with the original text; the same author/recipient attribution is retained and the private transport item id is removed. -This addresses readable child-result delivery (#3907), not scheduling or decryption. Blank, -malformed, ciphertext-only and mixed unknown/encrypted content retains the existing fail-closed -path. Forward destinations never enable the option. The parser and encrypted-task recovery -owners are unchanged, and no broad content-schema validation or adapter-wide string conversion -is introduced. Mocked server fixtures cover parent, child, and parent-result continuation over -SSE and JSON while preserving actual tool-call/result pairs. +This addresses readable child-result delivery (#3907), not scheduling or decryption. Blank, malformed, +ciphertext-only and mixed unknown/encrypted content stays unlowered here; backend ciphertext is replaced by the +[omission marker](../subagents.md#routed-agent-message-ciphertext-egress) before it can reach a routed destination. +Forward destinations never enable the option. The parser and encrypted-task recovery owners are unchanged, and no +broad content-schema validation or adapter-wide string conversion is introduced. Mocked server fixtures cover +parent, child, and parent-result continuation over SSE and JSON while preserving actual tool-call/result pairs. OpenCode Go documents `gpt-5.6-luna` on `/zen/go/v1/responses` while sibling models use its Chat or Anthropic endpoints. The built-in preset therefore selects `openai-responses` only for Luna and diff --git a/tests/responses/responses-opaque-blob-recovery.test.ts b/tests/responses/responses-opaque-blob-recovery.test.ts index 87eb74f9232..d36e67e4837 100644 --- a/tests/responses/responses-opaque-blob-recovery.test.ts +++ b/tests/responses/responses-opaque-blob-recovery.test.ts @@ -155,9 +155,11 @@ function serializedOutboundWithEncryptedAgentMessage(): string { } /** - * What a routed destination receives on the retry: recovery has replaced the undecryptable - * part with an omission marker, which leaves the item entirely plaintext, so the adapter - * converts it into the public user message a routed Responses schema can accept. + * What a routed destination receives: the undecryptable part has been replaced with an omission + * marker, which leaves the item entirely plaintext, so the adapter converts it into the public + * user message a routed Responses schema can accept. Since #4454 that repair runs before the + * first dispatch rather than after an upstream rejection, so this is the FIRST body such a + * destination sees, not a retry. */ function recoveredAgentMessage(): Record { return { @@ -171,6 +173,19 @@ function recoveredAgentMessage(): Record { }; } +/** The function-output twin: the reactive repair still owns this item type. */ +function recoveredFunctionOutput(): Record { + return { + type: "function_call_output", + call_id: "call-encrypted-output", + output: [ + { type: "input_text", text: "[encrypted content omitted]" }, + { type: "input_text", text: "visible tool output" }, + { type: "input_image", image_url: "data:image/png;base64,AAAA", detail: "high" }, + ], + }; +} + function config(): OcxConfig { return { defaultProvider: "first", @@ -249,6 +264,43 @@ function agentMessageRequest(stream = false): Request { }); } +/** + * The canonical Codex backend is exempt from the pre-dispatch repair (#4454), because it is the + * one destination that minted this ciphertext and can read it. That keeps + * `prepareOpaqueBlobRecovery`'s `agent_message` arm live exactly where it still makes sense: the + * backend failing to decrypt its own bytes. + */ +function nativeConfig(): OcxConfig { + return { + defaultProvider: "openai", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + } as OcxConfig; +} + +function nativeAgentMessageRequest(stream = false): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { + "content-type": "application/json", + "x-codex-parent-thread-id": "thread-native-encrypted-agent-message", + authorization: "Bearer caller-codex-token", + }, + body: JSON.stringify({ + model: "gpt-5.5", + stream, + store: false, + input: agentMessageReplayInput(), + }), + }); +} + function decryptStreamResponse(wire: string, contentType: string | null): Response { // A string body would implicitly add text/plain even when headers are omitted. const response = new Response(new TextEncoder().encode(wire), { @@ -548,11 +600,15 @@ describe("opaque blob recovery through /v1/responses", () => { }); }); - test("retries a ChatGPT agent-message decrypt failure once with an omission marker", async () => { + test("omits agent-message ciphertext before the first dispatch, with no decrypt round trip", async () => { + // This used to send the blob, collect `502 could not be decrypted`, repair, and retry. A + // routed destination was never going to decrypt a ChatGPT-minted blob, so the repair now runs + // first and the rejection never happens (#4454). The transient-5xx retry below is unrelated + // and still carries the already-repaired body. const outbound: Array> = []; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { outbound.push(JSON.parse(String(init?.body)) as Record); - return outbound.length <= 3 + return outbound.length <= 1 ? new Response(CHATGPT_FUNCTION_OUTPUT_DECRYPT_ERROR, { status: 502, headers: { "content-type": "application/json" }, @@ -565,31 +621,79 @@ describe("opaque blob recovery through /v1/responses", () => { expect(response.status).toBe(200); await response.text(); + expect(outbound).toHaveLength(2); + for (const sent of outbound) { + const input = sent.input as Array>; + expect(input.at(0)).toEqual(recoveredAgentMessage()); + expect(JSON.stringify(sent)).not.toContain(FUNCTION_OUTPUT_BLOB); + } + expect((outbound.at(1)?.input as Array>).at(1)) + .toEqual(agentMessageReplayInput().at(1)); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["transient-5xx"]); + }); + + test("omits agent-message ciphertext before the first streamed dispatch", async () => { + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return streamedSuccess("resp-stream-agent-message-repaired"); + }) as typeof fetch; + + const response = await handleResponses(agentMessageRequest(true), config(), { model: "", provider: "" }); + const body = await response.text(); + + expect(response.status).toBe(200); + expect(body).toContain("response.completed"); + expect(outbound).toHaveLength(1); + const sentInput = outbound.at(0)?.input as Array> | undefined; + expect(sentInput?.at(0)).toEqual(recoveredAgentMessage()); + expect(JSON.stringify(outbound.at(0))).not.toContain(FUNCTION_OUTPUT_BLOB); + }); + + test("still retries a ChatGPT agent-message decrypt failure on the canonical backend", async () => { + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return outbound.length <= 3 + ? new Response(CHATGPT_FUNCTION_OUTPUT_DECRYPT_ERROR, { + status: 502, + headers: { "content-type": "application/json" }, + }) + : success("resp-native-agent-message-recovered"); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(nativeAgentMessageRequest(), nativeConfig(), logCtx); + expect(response.status).toBe(200); + await response.text(); + expect(outbound).toHaveLength(4); - const retriedInput = outbound.at(3)?.input as Array> | undefined; - expect(retriedInput?.at(0)).toEqual(recoveredAgentMessage()); - expect(retriedInput?.at(1)).toEqual(agentMessageReplayInput().at(1)); + // The blob reaches this destination, which is the point of the exemption, and only the + // post-rejection repair takes it back off the wire. + expect(JSON.stringify(outbound.at(0))).toContain(FUNCTION_OUTPUT_BLOB); + expect(JSON.stringify(outbound.at(3))).not.toContain(FUNCTION_OUTPUT_BLOB); + expect(JSON.stringify(outbound.at(3))).toContain("[encrypted content omitted]"); expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["transient-5xx", "opaque-blob-rejection"]); }); - test("recovers a zero-output streamed agent-message decrypt failure before client relay", async () => { + test("still hides a streamed agent-message decrypt failure from the client on the canonical backend", async () => { const outbound: Array> = []; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { outbound.push(JSON.parse(String(init?.body)) as Record); return outbound.length === 1 ? streamedFunctionOutputDecryptFailure() - : streamedSuccess("resp-stream-agent-message-recovered"); + : streamedSuccess("resp-native-stream-agent-message-recovered"); }) as typeof fetch; - const response = await handleResponses(agentMessageRequest(true), config(), { model: "", provider: "" }); + const response = await handleResponses(nativeAgentMessageRequest(true), nativeConfig(), { model: "", provider: "" }); const body = await response.text(); expect(response.status).toBe(200); expect(body).toContain("response.completed"); expect(body).not.toContain(FUNCTION_OUTPUT_DECRYPT_MESSAGE); expect(outbound).toHaveLength(2); - const retriedInput = outbound.at(1)?.input as Array> | undefined; - expect(retriedInput?.at(0)).toEqual(recoveredAgentMessage()); + expect(JSON.stringify(outbound.at(0))).toContain(FUNCTION_OUTPUT_BLOB); + expect(JSON.stringify(outbound.at(1))).not.toContain(FUNCTION_OUTPUT_BLOB); }); test("recovers a zero-output error-event decrypt failure before client relay", async () => { @@ -601,7 +705,9 @@ describe("opaque blob recovery through /v1/responses", () => { : streamedSuccess("resp-stream-error-event-recovered"); }) as typeof fetch; - const response = await handleResponses(agentMessageRequest(true), config(), { model: "", provider: "" }); + // Carried by the function-output fixture: an agent message reaches a routed destination with + // its ciphertext already omitted, so it no longer has a blob for the upstream to reject. + const response = await handleResponses(functionOutputRequest(true), config(), { model: "", provider: "" }); const body = await response.text(); expect(response.status).toBe(200); @@ -609,7 +715,7 @@ describe("opaque blob recovery through /v1/responses", () => { expect(body).not.toContain(FUNCTION_OUTPUT_DECRYPT_MESSAGE); expect(outbound).toHaveLength(2); const retriedInput = outbound.at(1)?.input as Array> | undefined; - expect(retriedInput?.at(0)).toEqual(recoveredAgentMessage()); + expect(retriedInput?.at(1)).toEqual(recoveredFunctionOutput()); }); for (const streamMode of ["legacy-tee", "eager-relay"] as const) { @@ -644,7 +750,7 @@ describe("opaque blob recovery through /v1/responses", () => { const terminals: string[] = []; let markTerminal!: () => void; const terminal = new Promise(resolve => { markTerminal = resolve; }); - const response = await handleResponses(agentMessageRequest(true), { + const response = await handleResponses(functionOutputRequest(true), { ...config(), streamMode, }, logCtx, { onNativePassthroughTerminal: status => { terminals.push(status); @@ -684,7 +790,7 @@ describe("opaque blob recovery through /v1/responses", () => { return sends === 1 ? streamedFunctionOutputDecryptErrorEvent(true) : streamedSuccess("resp-identity"); }, { preconnect: originalFetch.preconnect }); try { - const response = await handleResponses(agentMessageRequest(true), config(), { model: "", provider: "" }); + const response = await handleResponses(functionOutputRequest(true), config(), { model: "", provider: "" }); const body = await response.text(); expect(body).toContain("response.completed"); expect(body).not.toContain(FUNCTION_OUTPUT_DECRYPT_MESSAGE); @@ -736,7 +842,7 @@ describe("opaque blob recovery through /v1/responses", () => { : streamedSuccess("resp-missing-ct-error-event-recovered"); }) as typeof fetch; - const response = await handleResponses(agentMessageRequest(true), config(), { model: "", provider: "" }); + const response = await handleResponses(functionOutputRequest(true), config(), { model: "", provider: "" }); const body = await response.text(); expect(response.status).toBe(200); @@ -744,7 +850,7 @@ describe("opaque blob recovery through /v1/responses", () => { expect(body).not.toContain(FUNCTION_OUTPUT_DECRYPT_MESSAGE); expect(outbound).toHaveLength(2); const retriedInput = outbound.at(1)?.input as Array> | undefined; - expect(retriedInput?.at(0)).toEqual(recoveredAgentMessage()); + expect(retriedInput?.at(1)).toEqual(recoveredFunctionOutput()); }); test("absent Content-Type decrypt stream does not recover a non-stream request", async () => { diff --git a/tests/server/v2-agent-message-failfast.test.ts b/tests/server/v2-agent-message-failfast.test.ts index 0d2b77dcdd8..0e0dc66a24b 100644 --- a/tests/server/v2-agent-message-failfast.test.ts +++ b/tests/server/v2-agent-message-failfast.test.ts @@ -1,4 +1,6 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { resolveWireProtocolOverride } from "../../src/server/adapter-resolve"; +import { structurallyValidFernetTokens } from "../../src/server/responses/encrypted-payload"; import { handleResponses, hasUnreadableEncryptedAgentTask, @@ -506,3 +508,359 @@ describe("V2 routed agent-message ciphertext guard", () => { expect(forwardedBody).toContain(FERNET_TASK); }); }); + +/** + * #4454. The guard above asks whether the CURRENT worker task is readable, and reads only the + * tail item. The adapter asks whether EVERY part can be lowered onto a public message. An item + * that mixes readable text with ciphertext answers "readable" to the first and "not lowerable" + * to the second, so it passed the guard, kept its private `agent_message` type through the raw + * Responses passthrough, and reached the provider as backend ciphertext plus an item type only + * the Codex backend declares. Position is incidental: a replayed child result simply tends to + * sit mid-history, where the tail-only scan could never have seen it. + * + * The repair is the one the opaque-blob path already applies after an upstream rejection. It + * runs before dispatch here, because a destination that cannot accept the private item was + * never going to answer that request anyway. + */ +describe("routed Responses agent-message ciphertext repair", () => { + function routedResponsesConfig(): OcxConfig { + return { + port: 0, + defaultProvider: "relay", + providers: { + relay: { + adapter: "openai-responses", + baseUrl: "https://relay.example/v1", + authMode: "key", + apiKey: "test-relay-key", + }, + }, + } as OcxConfig; + } + + // The reported destination: the provider-wide adapter is the Chat wire, and the registry moves + // grok-4.6 onto the raw Responses passthrough for an OAuth caller speaking Responses. Reading + // `route.provider.adapter` would miss it, so the repair resolves the same wire override the + // adapter is built from. + function xaiOAuthResponsesConfig(): OcxConfig { + return { + port: 0, + defaultProvider: "xai", + providers: { + xai: { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "oauth" }, + }, + } as OcxConfig; + } + + function mixedChildResult(): Record { + return { + type: "agent_message", + author: "/root/child", + recipient: "/root", + content: [ + { type: "input_text", text: "the child finished the migration" }, + { type: "encrypted_content", encrypted_content: FERNET_TASK }, + ], + }; + } + + const userTurn = { type: "message", role: "user", content: [{ type: "input_text", text: "continue" }] }; + + function captureOutbound(model: string): () => string[] { + const bodies: string[] = []; + globalThis.fetch = (async (_input, init) => { + bodies.push(typeof init?.body === "string" ? init.body : ""); + return Response.json({ + id: "resp_repaired", + object: "response", + status: "completed", + model, + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; + return () => bodies; + } + + test("repairs a mixed child result replayed behind a later user turn", async () => { + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", [mixedChildResult(), userTurn]); + + expect(response.status).toBe(200); + expect(outbound()).toHaveLength(1); + const sent = outbound()[0]!; + expect(sent).not.toContain(FERNET_TASK); + expect(sent).not.toContain("gAAAA"); + expect(sent).not.toContain("agent_message"); + expect(sent).toContain("[encrypted content omitted]"); + // The readable half of the item survives: only the bytes nobody could read are replaced. + expect(sent).toContain("the child finished the migration"); + }); + + test("repairs the same shape at the tail, where the readability guard reports readable", async () => { + const input = [mixedChildResult()]; + // The gap itself: this is the guard that was supposed to be the boundary. + expect(hasUnreadableEncryptedAgentTask(input)).toBe(false); + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", input); + + expect(response.status).toBe(200); + expect(outbound()[0]).not.toContain(FERNET_TASK); + expect(outbound()[0]).not.toContain("agent_message"); + }); + + test("omits ciphertext that arrives as text rather than in an encrypted slot", async () => { + // #3021 saw a delegated reply reach the parent as raw `gAAAA...` text. The readability guard + // reports it readable, and the xAI lowering path would have forwarded it as prose. + const input = [{ type: "agent_message", author: "/root/child", recipient: "/root", content: FERNET_TASK }]; + expect(hasUnreadableEncryptedAgentTask(input)).toBe(false); + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", input); + + expect(response.status).toBe(200); + expect(outbound()[0]).not.toContain(FERNET_TASK); + expect(outbound()[0]).toContain("[encrypted content omitted]"); + }); + + test("the reported xAI destination resolves onto the raw Responses wire", () => { + // The repair has to see this destination as the passthrough it becomes, not as the Chat wire + // the provider row names. The dispatch itself needs an OAuth credential this fixture has no + // business minting, so the wire resolution is asserted directly. + const provider = xaiOAuthResponsesConfig().providers.xai!; + expect(resolveWireProtocolOverride("xai", "grok-4.6", provider, "responses").adapter) + .toBe("openai-responses"); + expect(provider.adapter).toBe("openai-chat"); + }); + + test("leaves a fully readable child result exactly as the adapter already lowered it", async () => { + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", [{ + type: "agent_message", + author: "/root/child", + recipient: "/root", + content: [{ type: "input_text", text: "the child finished the migration" }], + }, userTurn]); + + expect(response.status).toBe(200); + expect(outbound()[0]).toContain("the child finished the migration"); + expect(outbound()[0]).not.toContain("[encrypted content omitted]"); + expect(outbound()[0]).not.toContain("agent_message"); + }); + + test("leaves a translated Chat destination on its existing path", async () => { + // The private item never reaches that wire: the parser rebuilds the body from messages and + // drops an encrypted part outright, so there is nothing to repair and no marker to add. + let forwardedBody = ""; + globalThis.fetch = (async (_input, init) => { + forwardedBody = typeof init?.body === "string" ? init.body : ""; + return Response.json({ + id: "chatcmpl_routed", + object: "chat.completion", + model: "grok-4.5", + choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; + + const response = await post(routedConfig(), "xai/grok-4.5", [mixedChildResult(), userTurn]); + + expect(response.status).toBe(200); + expect(forwardedBody).toContain("the child finished the migration"); + expect(forwardedBody).not.toContain(FERNET_TASK); + expect(forwardedBody).not.toContain("[encrypted content omitted]"); + }); + + test("leaves a forward destination's private item and ciphertext untouched", async () => { + let forwardedBody = ""; + globalThis.fetch = (async (_input, init) => { + forwardedBody = typeof init?.body === "string" ? init.body : ""; + return Response.json({ + id: "resp_native_mixed", + object: "response", + status: "completed", + model: "gpt-5.5", + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; + + const response = await post(nativeConfig(), "gpt-5.5", [mixedChildResult(), userTurn], { + authorization: "Bearer caller-codex-token", + }); + + expect(response.status).toBe(200); + expect(forwardedBody).toContain(FERNET_TASK); + expect(forwardedBody).toContain("agent_message"); + }); + + test("repairs a noncanonical forward gateway, which is not the backend that minted the bytes", async () => { + // `authMode: "forward"` describes how this proxy treats credentials, not who is on the other + // end. Only the canonical Codex backend can read its own ciphertext, so a forward-configured + // gateway at somebody else's origin is a third party like any other. + const config = { + port: 0, + defaultProvider: "relayfwd", + providers: { + relayfwd: { adapter: "openai-responses", baseUrl: "https://relay.example/v1", authMode: "forward" }, + }, + } as OcxConfig; + const outbound = captureOutbound("relay-model"); + + const response = await post(config, "relayfwd/child-model", [mixedChildResult(), userTurn]); + + expect(response.status).toBe(200); + expect(outbound()[0]).not.toContain(FERNET_TASK); + expect(outbound()[0]).toContain("[encrypted content omitted]"); + }); + + test("repairs a combo child, which carries its own clone of the body", async () => { + // `concreteComboRequestBody` structuredClones the body per target, so a repair applied on the + // parent's own dispatch is invisible here. A combo target that resolves to a routed Responses + // wire has to run the repair itself or it sends the ciphertext the parent no longer does. + const config = { + port: 0, + defaultProvider: "relay", + providers: { + relay: { + adapter: "openai-responses", + baseUrl: "https://relay.example/v1", + authMode: "key", + apiKey: "test-relay-key", + }, + }, + combos: { routed: { strategy: "failover", targets: [{ provider: "relay", model: "child-model" }] } }, + } as OcxConfig; + const outbound = captureOutbound("relay-model"); + + const response = await post(config, "combo/routed", [mixedChildResult(), userTurn]); + + expect(response.status).toBe(200); + expect(outbound()).toHaveLength(1); + expect(outbound()[0]).not.toContain(FERNET_TASK); + expect(outbound()[0]).not.toContain("agent_message"); + expect(outbound()[0]).toContain("[encrypted content omitted]"); + }); + + test("repairs a run split across consecutive encrypted slots", async () => { + // Each half fails structural validation on its own and only the join is a real token. A + // matcher that judged slots individually would forward both halves. + const first = FERNET_TASK.slice(0, 60); + const second = FERNET_TASK.slice(60); + expect(structurallyValidFernetTokens(first)).toEqual([]); + expect(structurallyValidFernetTokens(second)).toEqual([]); + expect(structurallyValidFernetTokens(`${first}${second}`)).toEqual([FERNET_TASK]); + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", [{ + type: "agent_message", + author: "/root/child", + recipient: "/root", + content: [ + { type: "input_text", text: "Message Type: MESSAGE\nTask name: /root\nSender: /root/child\nPayload:" }, + { type: "encrypted_content", encrypted_content: first }, + { type: "encrypted_content", encrypted_content: second }, + ], + }, userTurn]); + + expect(response.status).toBe(200); + expect(outbound()[0]).not.toContain(first); + expect(outbound()[0]).not.toContain(second); + expect(outbound()[0]).not.toContain("agent_message"); + expect(outbound()[0]).toContain("[encrypted content omitted]"); + }); + + test("repairs a token embedded inside a text part and keeps the prose around it", async () => { + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", [{ + type: "agent_message", + author: "/root/child", + recipient: "/root", + content: [{ type: "input_text", text: `the child replied ${FERNET_TASK} and stopped` }], + }, userTurn]); + + expect(response.status).toBe(200); + expect(outbound()[0]).not.toContain(FERNET_TASK); + expect(outbound()[0]).toContain("the child replied [encrypted content omitted] and stopped"); + }); + + test("leaves readable text that only resembles an encoded blob", async () => { + // An `encrypted_content` slot carries ciphertext by definition, so it is stripped whatever it + // holds. A text part does not. Judging text by a loose character class would be worse than the + // defect for that half: a SHA-256 digest is exactly 64 characters of the same alphabet, and a + // child that deliberately printed one would have it silently deleted. + const readable = { + sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + sha512: "cf83e1357eefb8bd".repeat(8), + apiKey: `sk-proj-${"A".repeat(120)}`, + }; + for (const [name, text] of Object.entries(readable)) { + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", [{ + type: "agent_message", + author: "/root/child", + recipient: "/root", + content: [{ type: "input_text", text: `digest ${text}` }, { type: "input_text", text }], + }, userTurn]); + + expect(response.status, name).toBe(200); + expect(outbound()[0], name).toContain(text); + expect(outbound()[0], name).not.toContain("[encrypted content omitted]"); + } + }); + + test("repairs a token split across adjacent text parts", async () => { + // The text-side twin of the split encrypted slot. The join must still be Fernet-shaped, so + // two ordinary encoded fragments do not become a marker merely by being adjacent. + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", [{ + type: "agent_message", + author: "/root/child", + recipient: "/root", + content: [ + { type: "input_text", text: FERNET_TASK.slice(0, 60) }, + { type: "input_text", text: FERNET_TASK.slice(60) }, + ], + }, userTurn]); + + expect(response.status).toBe(200); + expect(outbound()[0]).not.toContain(FERNET_TASK.slice(0, 60)); + expect(outbound()[0]).toContain("[encrypted content omitted]"); + }); + + test("repairs a slot that is not a well-formed token, including standard base64", async () => { + // The original defect reached the wire because an item was not lowerable. Recognizing only + // canonical Fernet would reopen it one payload later: a truncated token, a bad version byte, + // or standard base64 carrying + and / would each keep the item and forward the bytes. + const nearMisses = { + truncated: FERNET_TASK.slice(0, 96), + standardBase64: `gAAA+${"B".repeat(120)}/x==`, + badVersion: `h${FERNET_TASK.slice(1)}`, + }; + for (const [name, blob] of Object.entries(nearMisses)) { + expect(structurallyValidFernetTokens(blob)).toEqual([]); + const outbound = captureOutbound("relay-model"); + + const response = await post(routedResponsesConfig(), "relay/child-model", [{ + type: "agent_message", + author: "/root/child", + recipient: "/root", + content: [ + { type: "input_text", text: "visible child result" }, + { type: "encrypted_content", encrypted_content: blob }, + ], + }, userTurn]); + + expect(response.status, name).toBe(200); + expect(outbound()[0], name).not.toContain(blob); + expect(outbound()[0], name).not.toContain("agent_message"); + expect(outbound()[0], name).toContain("visible child result"); + } + }); +});