From d210509d029e53767f14239394e1aefaae2e6dfd Mon Sep 17 00:00:00 2001 From: Riccardo Palleschi Date: Sun, 20 Sep 2026 13:30:33 +0200 Subject: [PATCH] fix(server): send Codex browser guidance as application context --- .../provider/CodexDeveloperInstructions.ts | 16 +-- .../Layers/CodexSessionRuntime.test.ts | 111 +++++++++++++----- .../provider/Layers/CodexSessionRuntime.ts | 13 +- 3 files changed, 100 insertions(+), 40 deletions(-) diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index 6a7fee351bce..1dcba7aec55e 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -36,12 +36,12 @@ const normalizeAvailability = ( * from Playwright, agent-browser, and raw simctl/adb, so leaving them in would * talk it out of the only automation it still has. */ -const browserToolInstructions = (availability: boolean | T3CodeToolAvailability): string => { - const tools = normalizeAvailability(availability); - return `${tools.browser ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : ""}${ - tools.device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : "" - }`; -}; +export const buildCodexBrowserInstructions = ( + availability: boolean | T3CodeToolAvailability, +): string => (normalizeAvailability(availability).browser ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : ""); + +const deviceToolInstructions = (availability: boolean | T3CodeToolAvailability): string => + normalizeAvailability(availability).device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : ""; const codexPlanModeDeveloperInstructions = ( browserToolsAvailable: boolean | T3CodeToolAvailability, @@ -173,7 +173,7 @@ Do not ask "should I proceed?" in the final output. The user can easily switch o Only produce at most one \`\` block per turn, and only when you are presenting a complete spec. If the user stays in Plan mode and asks for revisions after a prior \`\`, any new \`\` must be a complete replacement. If the user indicates that the prior plan is not acceptable but does not provide enough information to produce a complete replacement, address the concern and continue planning without producing a \`\` block. If the follow-up neither requires changes nor calls the plan into question (e.g. clarifying question), answer it before the block, then reproduce the prior \`\` unchanged. -${browserToolInstructions(browserToolsAvailable)} +${deviceToolInstructions(browserToolsAvailable)} `; const codexDefaultModeDeveloperInstructions = ( @@ -189,7 +189,7 @@ Your active mode changes only when new developer instructions with a different \ Use the \`request_user_input\` tool only when it is listed in the available tools for this turn. In Default mode, strongly prefer making reasonable assumptions and executing the user's request rather than stopping to ask questions. If you absolutely must ask a question because the answer cannot be discovered from local context and a reasonable assumption would be risky, ask the user directly with a concise plain-text question. Never write a multiple choice question as a textual assistant message. -${browserToolInstructions(browserToolsAvailable)} +${deviceToolInstructions(browserToolsAvailable)} `; export interface CodexRuntimeInfo { diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts index ec113ab7c521..6242d2a6c4a6 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts @@ -9,7 +9,10 @@ import * as CodexErrors from "effect-codex-app-server/errors"; import * as CodexRpc from "effect-codex-app-server/rpc"; import * as EffectCodexSchema from "effect-codex-app-server/schema"; -import { buildCodexDeveloperInstructions } from "../CodexDeveloperInstructions.ts"; +import { + buildCodexBrowserInstructions, + buildCodexDeveloperInstructions, +} from "../CodexDeveloperInstructions.ts"; import { codexSessionAppServerArgs } from "./codexLaunchArgs.ts"; import { buildTurnStartParams, @@ -154,6 +157,72 @@ function makeThreadOpenResponse( } describe("buildTurnStartParams", () => { + it.effect("delivers browser guidance as application context without changing user input", () => + Effect.gen(function* () { + for (const interactionMode of [undefined, "default", "plan"] as const) { + const params = yield* buildTurnStartParams({ + threadId: "provider-thread-1", + runtimeMode: "full-access", + prompt: "Check the UI", + attachments: [{ type: "localImage", path: "/tmp/page.png" }], + ...(interactionMode ? { interactionMode } : {}), + browserToolsAvailable: { browser: true, device: true }, + }); + NodeAssert.deepEqual(params.input, [ + { type: "text", text: "Check the UI" }, + { type: "localImage", path: "/tmp/page.png" }, + ]); + NodeAssert.equal(params.additionalContext?.t3_browser?.kind, "application"); + NodeAssert.match( + params.additionalContext?.t3_browser?.value ?? "", + /first call `preview_status`/, + ); + NodeAssert.doesNotMatch( + params.collaborationMode?.settings.developer_instructions ?? "", + /T3 Code collaborative browser/, + ); + if (interactionMode) { + NodeAssert.match( + params.collaborationMode?.settings.developer_instructions ?? "", + /device_list/, + ); + } + } + }), + ); + + it.effect("omits browser context unless the preview tools are attached", () => + Effect.gen(function* () { + for (const browserToolsAvailable of [undefined, false, { browser: false, device: true }]) { + const params = yield* buildTurnStartParams({ + threadId: "provider-thread-1", + runtimeMode: "full-access", + prompt: "Check the app", + ...(browserToolsAvailable !== undefined ? { browserToolsAvailable } : {}), + }); + NodeAssert.equal(params.additionalContext, undefined); + } + }), + ); + + it.effect("keeps browser context stable across mode changes and image-only turns", () => + Effect.gen(function* () { + const contexts = []; + for (const interactionMode of ["default", "plan"] as const) { + const params = yield* buildTurnStartParams({ + threadId: "provider-thread-1", + runtimeMode: "full-access", + attachments: [{ type: "localImage", path: "/tmp/page.png" }], + interactionMode, + browserToolsAvailable: true, + }); + contexts.push(params.additionalContext); + NodeAssert.deepEqual(params.input, [{ type: "localImage", path: "/tmp/page.png" }]); + } + NodeAssert.deepEqual(contexts[0], contexts[1]); + }), + ); + it.effect("sends currency skill aliases in Codex's canonical dollar form", () => Effect.gen(function* () { for (const symbol of ["€", "£", "¥", "₹", "₩", "₿", "𑿝"]) { @@ -617,40 +686,20 @@ describe("buildCodexDeveloperInstructions", () => { }); describe("T3 browser developer instructions", () => { - const runtime = { model: "gpt-5.3-codex", reasoningEffort: "high" }; - - it("prefers the product-native preview tools in both collaboration modes", () => { - for (const mode of ["default", "plan"] as const) { - const instructions = buildCodexDeveloperInstructions(mode, runtime, true); - NodeAssert.match(instructions, /t3-code/); - NodeAssert.match(instructions, /preview_status/); - NodeAssert.match(instructions, /preview_open/); - NodeAssert.match(instructions, /Do not switch to global browser skills/); - } + it("retains preview initialization, error recovery, and explicit browser overrides", () => { + const instructions = buildCodexBrowserInstructions({ browser: true, device: false }); + NodeAssert.match(instructions, /first call `preview_status`/); + NodeAssert.match(instructions, /call `preview_open` before concluding/); + NodeAssert.match(instructions, /user explicitly requests another browser/); + NodeAssert.match(instructions, /retried with corrected arguments/); + NodeAssert.doesNotMatch(instructions, /|/); }); - it("omits the browser block entirely when the preview tools are not attached", () => { - for (const mode of ["default", "plan"] as const) { - const instructions = buildCodexDeveloperInstructions(mode, runtime, false); - NodeAssert.doesNotMatch(instructions, /preview_status/); - NodeAssert.doesNotMatch(instructions, /preview_open/); - NodeAssert.doesNotMatch(instructions, /T3 Code collaborative browser/); - // Steering away from other browser automation must go with the tools; - // keeping it would leave the model talked out of its only option. - NodeAssert.doesNotMatch(instructions, /Do not switch to global browser skills/); - // The rest of the collaboration mode is untouched. - NodeAssert.match(instructions, //); - NodeAssert.match(instructions, /<\/collaboration_mode>/); + it("omits browser guidance when disabled or only device tools are attached", () => { + for (const availability of [false, { browser: false, device: true }]) { + NodeAssert.equal(buildCodexBrowserInstructions(availability), ""); } }); - - it("tracks the turn's MCP configuration rather than defaulting to on", () => { - NodeAssert.match(buildCodexDeveloperInstructions("default", runtime, true), /preview_open/); - NodeAssert.doesNotMatch( - buildCodexDeveloperInstructions("default", runtime, false), - /preview_open/, - ); - }); }); describe("hasConfiguredMcpServer", () => { diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 674d23327b65..bb228d6910d0 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -40,6 +40,7 @@ import { buildCodexInitializeParams } from "./CodexProvider.ts"; import { codexSessionAppServerArgs } from "./codexLaunchArgs.ts"; import { expandHomePath } from "../../pathExpansion.ts"; import { + buildCodexBrowserInstructions, buildCodexDeveloperInstructions, type T3CodeToolAvailability, } from "../CodexDeveloperInstructions.ts"; @@ -145,6 +146,9 @@ const isMcpElicitationForm = Schema.is(McpElicitationForm); const CodexTurnStartParamsWithCollaborationMode = EffectCodexSchema.V2TurnStartParams.pipe( Schema.fieldsAssign({ collaborationMode: Schema.optionalKey(EffectCodexSchema.V2TurnStartParams__CollaborationMode), + additionalContext: Schema.optionalKey( + Schema.Record(Schema.String, EffectCodexSchema.V2TurnStartParams__AdditionalContextEntry), + ), }), ); const decodeCodexTurnStartParamsWithCollaborationMode = Schema.decodeUnknownEffect( @@ -621,7 +625,6 @@ export function buildTurnStartParams(input: { readonly serviceTier?: CodexServiceTier; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; readonly interactionMode?: ProviderInteractionMode; - /** Defaults to true so callers that predate the agent-access gate are unchanged. */ readonly browserToolsAvailable?: boolean | T3CodeToolAvailability; }): Effect.Effect< CodexTurnStartParamsWithCollaborationMode, @@ -646,9 +649,17 @@ export function buildTurnStartParams(input: { browserToolsAvailable: input.browserToolsAvailable ?? true, }); + const browserInstructions = buildCodexBrowserInstructions(input.browserToolsAvailable ?? false); return decodeCodexTurnStartParamsWithCollaborationMode({ threadId: input.threadId, input: turnInput, + ...(browserInstructions + ? { + additionalContext: { + t3_browser: { kind: "application", value: browserInstructions }, + }, + } + : {}), approvalPolicy: config.approvalPolicy, approvalsReviewer: config.approvalsReviewer, sandboxPolicy: runtimeModeToTurnSandboxPolicy(input.runtimeMode),