diff --git a/.agents/skills/test-t3-app/SKILL.md b/.agents/skills/test-t3-app/SKILL.md index 6fde6fff30984..7ac0eb1106b32 100644 --- a/.agents/skills/test-t3-app/SKILL.md +++ b/.agents/skills/test-t3-app/SKILL.md @@ -5,10 +5,10 @@ description: Test T3 Code's web and desktop UI through its built-in Browser pane # Test T3 web and desktop -Use T3's built-in Browser panel for verification. If its tools are absent or -the panel reports unavailable, explain the blocker and stop verification. -Do not install or switch to another automation system. For native mobile -testing, use [test-t3-mobile](../test-t3-mobile/SKILL.md). +Use T3's built-in Browser panel for verification. Follow the recovery steps +below before declaring it unavailable. Do not install or switch to another +automation system. For native mobile testing, use +[test-t3-mobile](../test-t3-mobile/SKILL.md). ## Start the app @@ -36,8 +36,13 @@ model, such as Haiku for Claude or Luna for Codex. ## Use the Browser panel -Call `preview_status`, then `preview_open` if the Browser panel is -closed. Navigate to the complete startup pairing URL once with +Call `preview_status`. If no tab can be automated, call `preview_open`, including +when status reports `available: false` or `tabId: null`. Status can report +unavailable simply because no tab exists yet; opening a tab needs no user action. +Stop verification and explain the blocker only if the tools are absent or +`preview_open` reports unavailable or fails. + +Navigate to the complete startup pairing URL once with `preview_navigate`, then use `preview_snapshot` and T3's interaction tools. If the token was consumed or expired, run `node apps/server/src/bin.ts pair` for a fresh one. Keep using the same tab. diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index 3b817f574455f..fadea6438da46 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -26,6 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work. - `experience`: the real mistake, in a sentence. `source`: where it happened. - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. +- `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt. - `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Check that the case builds with `--dry-run --case `. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass. diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson index 1940692d9fe17..1b9d713307e91 100644 --- a/evals/mcp-tool-use/cases/visual-preview.ndjson +++ b/evals/mcp-tool-use/cases/visual-preview.ndjson @@ -28,3 +28,5 @@ {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}} {"id": "requested-after-video-for-edit", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "agent refuses requested browser recording of its own edit", "experience": "A user asked for an after video of a UI fix. The agent declined because the injected instructions prohibited preview for its own edits.", "rule": "Use preview when the user asks for browser work, including recording an edit.", "setup": {"role": "top", "context": "The UI fix is complete and a development server is already running at http://localhost:3000. Browser verification and recording are authorized.", "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_recording_start": {"recording": true, "tabId": "tab_1", "startedAt": "2026-01-01T00:00:00.000Z"}, "preview_recording_stop": {"id": "recording_1", "tabId": "tab_1", "path": "/tmp/after.webm", "mimeType": "video/webm", "sizeBytes": 1024, "createdAt": "2026-01-01T00:00:01.000Z"}}}, "prompt": "Use the T3 browser to verify the UI fix you just made and record an after video.", "expect": {"calls": [{"tool": "preview_recording_start"}, {"tool": "preview_recording_stop"}], "order": ["preview_recording_start", "preview_recording_stop"]}, "source": {"report": "A user requested an after video of a UI fix; the agent declined because preview was prohibited for its own edits."}} {"id": "dev-server-opened-blank-then-navigated", "area": "visual-preview", "kind": "bad", "pattern": "preview_open on a blank tab, then preview_navigate to the dev-server port", "experience": "Asked to open a dev server in the T3 preview, agents opened a blank tab and navigated it to the port in a second call. 121 of these pairs in a month of maintainer threads.", "rule": "preview_open takes a url but not a dev-server target", "privacy": "commit-safe", "setup": {"role": "top", "context": "CLAUDE.md says: for t3code browser work, use the server-side preview_* mcp tools, with `target: {kind: \"environment-port\", port: ...}` for dev servers in the current environment. The dev server for this repo is running; `vp run dev` printed `Local: http://localhost:5173/`.", "mocks": {"preview_status": {"available": true, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "open the dev server in the t3 preview, i want to click around", "expect": {"calls": [{"tool": "preview_open", "args": {"target.port": 5173}}], "maxCalls": {"preview_open": 1, "preview_navigate": 0}}, "maxSteps": 6, "source": {"thread": "Maintainer threads, 2026-10: preview_open with no url followed by preview_navigate to an environment-port target, 121 times"}} +{"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} +{"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return ; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 217e505f5fcf5..6c24f18ac5063 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -394,6 +394,28 @@ it("replays earlier capability results with the model under test as the caller", .flatMap((message) => message.content) .find((block) => block.type === "tool_result" && String(block.content).includes("inherited")); assert.include(String(replayed?.content), '"inheritedModel":"claude-opus-5-5"'); + const resumed = buildMessages(makeSample({ ...testCase, prompt: "" }, base)); + assert.equal(resumed.at(-1)?.role, "user"); + assert.equal(resumed.at(-1)?.content.at(-1)?.type, "tool_result"); + for (const setup of [ + { + ...testCase.setup, + sideChat: { parentThreadId: "thread-parent", digest: ["The permissions fix is committed."] }, + }, + { ...testCase.setup, role: "native-subagent" as const }, + { ...testCase.setup, role: "delegated-child" as const }, + ]) { + assert.throws( + () => + buildMessages( + makeSample( + { ...testCase, prompt: "", setup }, + { ...base, subagentContext: "Hook context" }, + ), + ), + /Empty-prompt replay/, + ); + } }); it("gives each launched thread its own id, and reading it back shows that thread's brief", () => { diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 3e6bb84f697be..02c5d8ce58e46 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -140,6 +140,8 @@ const PriorToolCall = Schema.Struct({ input: Schema.Unknown, /** For a device tool, `"default"` runs the call on the world's devices for T3's result. */ result: Schema.Unknown, + /** Repository-relative file whose current contents supply this tool result. */ + resultFile: Schema.optional(Schema.String), }); /** @@ -275,6 +277,7 @@ export const McpToolUseCase = Schema.Struct({ Schema.Struct({ parentThreadId: Schema.String, digest: Schema.Array(Schema.String) }), ), }), + /** Empty to resume a top-level thread without side-chat context after its last tool result. */ prompt: Schema.String, /** Records for the context chips `prompt` links, projected as T3 sends them. */ promptContext: Schema.optional(Schema.Struct({ records: Schema.Array(ComposerContextRecord) })), @@ -880,6 +883,19 @@ export const buildMessages = (sample: Sample) => { else messages[messages.length - 1] = { role, content: [...last.content, ...blocks] }; }; const { role, context, delegateRole, priorTurns = [] } = testCase.setup; + const sideChat = testCase.setup.sideChat; + if (testCase.prompt === "" && (role !== "top" || sideChat !== undefined)) { + throw new Error( + `${testCase.id}: Empty-prompt replay requires a top-level thread without side-chat context.`, + ); + } + const sideChatContext = + sideChat === undefined + ? undefined + : makeSideChatContext({ + parentThreadId: ThreadId.make(sideChat.parentThreadId), + digest: sideChat.digest, + }); const native = role === "native-subagent"; const notes = [ native @@ -926,19 +942,11 @@ export const buildMessages = (sample: Sample) => { testCase.promptContext === undefined ? testCase.prompt : projectComposerContextForProvider({ text: testCase.prompt, ...testCase.promptContext }); - const sideChat = testCase.setup.sideChat; const prompt = - sideChat === undefined - ? userText - : withInlineContext( - makeSideChatContext({ - parentThreadId: ThreadId.make(sideChat.parentThreadId), - digest: sideChat.digest, - }), - userText, - ); + sideChatContext === undefined ? userText : withInlineContext(sideChatContext, userText); const childPrompt = () => taskPrompt({ task: prompt, role: delegateRole }); - push("user", text(role === "delegated-child" ? childPrompt() : prompt)); + if (testCase.prompt !== "") + push("user", text(role === "delegated-child" ? childPrompt() : prompt)); // Claude Code gives a subagent its SubagentStart hooks' context as a reminder after the task. if (native && base.subagentContext) { push("user", reminder(`SubagentStart hook additional context: ${base.subagentContext}`)); @@ -2125,12 +2133,35 @@ const main = Effect.gen(function* () { const wanted = flags.case?.flatMap((ids) => ids.split(",")); const codex = isCodex(flags.model); const harness = codex ? "codex" : "claude"; - const selected = cases.filter( + const selectedCases = cases.filter( ({ id, setup }) => (wanted === undefined || wanted.includes(id)) && (setup.harness === undefined || setup.harness === harness), ); - if (selected.length === 0) return yield* Effect.die(new Error("No case matches --case.")); + if (selectedCases.length === 0) return yield* Effect.die(new Error("No case matches --case.")); + const selected = yield* Effect.forEach(selectedCases, (testCase) => + Effect.gen(function* () { + const priorTurns = yield* Effect.forEach(testCase.setup.priorTurns ?? [], (turn) => + Effect.gen(function* () { + const toolCalls = yield* Effect.forEach(turn.toolCalls ?? [], (call) => + call.resultFile === undefined + ? Effect.succeed(call) + : fs + .readFileString(path.resolve(import.meta.dirname, "../..", call.resultFile)) + .pipe(Effect.map((result) => ({ ...call, result }))), + ); + return { ...turn, toolCalls }; + }), + ); + return { + ...testCase, + setup: { + ...testCase.setup, + priorTurns, + }, + }; + }), + ); // Trimmed captures without user data ship beside the cases. `--template` also takes a raw // CLIProxyAPI capture. const templatePath =