From 122524e561b7a9ece5a491425989955f4778c50f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 7 Oct 2026 13:55:24 -0700 Subject: [PATCH 1/7] fix(server): browser snapshots leave the screenshot out of tool history by default preview_snapshot now returns an image only with includeImage=true, and save=true returns just the url and saved path. Providers replay inline images on every later request, and some reject them outright. Fixes #11295 Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/mcp/McpHttpServer.test.ts | 19 +++++++++---------- apps/server/src/mcp/McpHttpServer.ts | 11 +++++++---- apps/server/src/mcp/toolkits/preview/tools.ts | 6 +++--- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 3505e4e58633..a057605688b8 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -244,7 +244,7 @@ it.effect("tells the agent how to fall back when no desktop app can run the snap ); it.effect.each([ - { mode: "default", input: {}, images: true }, + { mode: "default", input: {}, images: false }, { mode: "explicit image", input: { includeImage: true }, images: true }, { mode: "text only", input: { includeImage: false }, images: false }, ])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) => @@ -351,12 +351,7 @@ it.effect.each([ Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provideService(McpSchema.McpServerClient, client), ); - expect(nextDefault.content.map((content) => content.type)).toEqual([ - "text", - "text", - "text", - "image", - ]); + expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "text"]); expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot }); expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree"); expect(requests).toBe(7); @@ -416,11 +411,12 @@ it.effect("saves the snapshot PNG on request and reports its path", () => const path = yield* Path.Path; const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult); - const snapshot = yield* callSnapshot({ save: true }); + const snapshot = yield* callSnapshot({ save: true, includeImage: true }); expect(snapshot.isError).toBe(false); // The browser never receives the server-only `save` flag. expect(inputs).toEqual([{}]); + expect(snapshot.content.map((content) => content.type)).toContain("image"); const structured = snapshot.structuredContent as { readonly screenshotPath?: string }; const screenshotPath = structured.screenshotPath; expect(typeof screenshotPath).toBe("string"); @@ -436,7 +432,7 @@ it.effect("saves the snapshot PNG on request and reports its path", () => expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath"); // A save without the image skips the page dump. - const pathOnly = yield* callSnapshot({ save: true, includeImage: false }); + const pathOnly = yield* callSnapshot({ save: true }); const saved = pathOnly.structuredContent as { readonly screenshotPath: string }; expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) }); expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png"); @@ -891,7 +887,10 @@ it.effect("registers annotated tools and preserves authenticated request context expect(malformed._tag).toBe("InvalidParams"); const snapshot = yield* server - .callTool({ name: "preview_snapshot", arguments: { tabId: alternateTabId } }) + .callTool({ + name: "preview_snapshot", + arguments: { tabId: alternateTabId, includeImage: true }, + }) .pipe( Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provideService(McpSchema.McpServerClient, client), diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index b63cd7b95d7e..f31c1576f5ae 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -503,7 +503,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); const screenshotPath = payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; - if (screenshotPath !== undefined && payload?.includeImage === false) { + // Images stay out of tool history unless asked for: providers replay them on every + // later request, and some reject inline images outright. + const includeImage = payload?.includeImage === true; + if (screenshotPath !== undefined && !includeImage) { // The agent only wants a file to show the user. The url keeps the site icon on the tool row. const saved = { url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), @@ -548,9 +551,9 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, }, ]), - ...(payload?.includeImage === false - ? [] - : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), + ...(includeImage + ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }] + : []), ], }); }), diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 9c94a1d446c6..b35216fe4878 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -142,19 +142,19 @@ const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = readonlyBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and screenshot dimensions. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( Schema.Boolean.annotate({ description: - "Include the PNG image in the tool response. Defaults to true. Set false for text-only output.", + "Include the PNG image in the tool response. Defaults to false. Set true only when you need to see the page.", }), ), save: Schema.optional( Schema.Boolean.annotate({ description: - "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.", + "Write the screenshot PNG to disk and return its absolute path as screenshotPath. Unless includeImage=true, return only the url and screenshotPath. Defaults to false.", }), ), }), From 6ddd95fddd968df0792ffa5fa880895612031edf Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 7 Oct 2026 13:56:46 -0700 Subject: [PATCH 2/7] fix(server): agents fall back from a failing preview without asking, and follow invoked browser workflows The injected browser rule keeps the T3 preview as the default, but now lets an agent switch after two failed preview calls on the same step (quoting the raw error, without asking which browser to use), and defers to a skill or documented repository workflow the user invoked. Fixes #16404 Fixes #15798 Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/provider/T3OrchestrationInstructions.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/server/src/provider/T3OrchestrationInstructions.ts b/apps/server/src/provider/T3OrchestrationInstructions.ts index fe700c0e2084..6412991f34e7 100644 --- a/apps/server/src/provider/T3OrchestrationInstructions.ts +++ b/apps/server/src/provider/T3OrchestrationInstructions.ts @@ -43,7 +43,10 @@ You are running inside T3 Code. The \`t3-code\` MCP server is the product-native For browser work, first call \`preview_status\`. If no automation-capable preview is attached, call \`preview_open\` before concluding that the browser is unavailable. Then use \`preview_navigate\`, \`preview_snapshot\`, and the focused interaction tools. Prefer snapshot-provided locators over coordinates. -Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Use an alternative browser system only when the T3 preview tools are absent, the user explicitly requests another browser, or \`preview_open\` returns an explicit unsupported/unavailable error. A failed T3 preview tool call should be inspected and retried with corrected arguments when the error is actionable. +Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Inspect a failed preview call and retry with corrected arguments when the error is actionable. Use another browser system when: +- the T3 preview tools are absent, or \`preview_open\` returns an explicit unsupported/unavailable error; +- the user asks for another browser, or invokes a skill or documented repository workflow that names one; follow that workflow and report any prerequisite it is missing; +- preview calls on an open tab have failed twice on the same step (timeouts, \`chrome-error://\` pages, a different client answering). Quote the raw error and switch without asking the user which browser to use. `; const T3_CODE_ACP_DEFAULT_MODE_INSTRUCTIONS = `## T3 Code interaction mode: Default From cf1890df2b2bda9d559affa6b88e6fbc888d4405 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 7 Oct 2026 14:00:42 -0700 Subject: [PATCH 3/7] fix(web): agent-opened browser tabs use the default browser profile preview_open sends no profile, and the desktop mounted those tabs in the built-in Default partition. Tabs without a profile now fall back to the profile chosen in Settings, latched once so a live tab never changes partition. Fixes #16701 Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/src/browser/ElectronBrowserHost.tsx | 24 ++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index 5cbefd9dded6..1c3be69ad779 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -3,16 +3,18 @@ import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; import { AuthPreviewOperateScope, FILL_PREVIEW_VIEWPORT } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; -import { type ComponentProps, useEffect, useMemo } from "react"; +import { type ComponentProps, useEffect, useMemo, useState } from "react"; import { primaryEnvironmentIdAtom } from "~/state/primaryEnvironment"; import { isElectron } from "~/env"; +import { useClientSettingsHydrated } from "~/hooks/useSettings"; import { useTheme } from "~/hooks/useTheme"; import { useActivePreviewSessions } from "~/previewStateStore"; import { useEnvironmentScope } from "~/state/session"; import { readPreviewAnnotationTheme } from "./annotationTheme"; +import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; import { HostedBrowserWebview } from "./HostedBrowserWebview"; import { rendersServerTabNatively } from "./previewRuntime"; @@ -129,5 +131,23 @@ function AuthorizedBrowserWebview(props: ComponentProps : null; + const profileId = useTabProfileId(props.profileId); + return canOperatePreview ? : null; +} + +/** + * Tabs opened without a profile (agent `preview_open`) use the configured + * default, like human opens. The server cannot read client settings, so the + * fallback happens here. It is latched once settings load: Electron fixes the + * partition when the guest attaches, so a later settings change must not move + * a live tab. + */ +function useTabProfileId(profileId: string | undefined): string | undefined { + const hydrated = useClientSettingsHydrated(); + const defaultProfileId = useBrowserDefaults().profileId; + const [fallback, setFallback] = useState(undefined); + if (profileId === undefined && hydrated && fallback === undefined) { + setFallback(defaultProfileId); + } + return profileId ?? fallback; } From 0cc053daf730116974f1fa13e960a89f762265ac Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 7 Oct 2026 14:09:16 -0700 Subject: [PATCH 4/7] fix(server): MCP device screenshots and upload discards check who owns them device_screenshot with an explicit deviceId now only reaches devices the calling thread opened, t3_attachment_discard only discards uploads the same caller prepared, and preview_snapshot is no longer annotated read-only since save=true writes a file. Refs #15962 Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/mcp/McpDeviceToolkit.test.ts | 23 ++- apps/server/src/mcp/McpHttpServer.test.ts | 4 +- apps/server/src/mcp/McpHttpServer.ts | 4 +- .../src/mcp/toolkits/attachment/handlers.ts | 140 +++++++++++------- apps/server/src/mcp/toolkits/core.test.ts | 50 +++++++ .../src/mcp/toolkits/device/handlers.ts | 19 ++- apps/server/src/mcp/toolkits/preview/tools.ts | 3 +- 7 files changed, 180 insertions(+), 63 deletions(-) diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts index 3c2f4e5905eb..a77e8f600252 100644 --- a/apps/server/src/mcp/McpDeviceToolkit.test.ts +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { DeviceHostUnavailableError, + DeviceId, EnvironmentId, ProviderInstanceId, ThreadId, @@ -93,7 +94,21 @@ const layerDeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ platform: input.platform, openedAt: "2026-09-08T00:00:00.000Z", }), - sessionsForThread: () => Effect.succeed([]), + // UDID-1 is open in the test thread; UDID-2 exists but belongs to another thread. + sessionsForThread: (id) => + Effect.succeed( + id === threadId + ? [ + { + threadId, + hostId: "local", + deviceId: DeviceId.make("UDID-1"), + platform: "ios" as const, + openedAt: "2026-09-08T00:00:00.000Z", + }, + ] + : [], + ), screenshot: () => Effect.succeed({ device, png }), close: () => Effect.void, agentCli: Effect.succeed("/cli"), @@ -135,6 +150,12 @@ it.effect("registers the device tools and returns the screenshot as image conten screenshot: { mimeType: "image/png", width: 1206, height: 2622 }, }); + const foreign = yield* server + .callTool({ name: "device_screenshot", arguments: { deviceId: "UDID-2" } }) + .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(foreign.isError).toBe(true); + expect(foreign.content.map((entry) => entry.type)).toEqual(["text"]); + const denied = yield* server .callTool({ name: "device_list", arguments: {} }) .pipe(callWith(["preview"]), Effect.provideService(McpSchema.McpServerClient, client)); diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index a057605688b8..fb9ca151d360 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -847,8 +847,8 @@ it.effect("registers annotated tools and preserves authenticated request context expect(statusTool?.tool.annotations?.destructiveHint).toBe(false); const snapshotTool = server.tools.find(({ tool }) => tool.name === "preview_snapshot"); - expect(snapshotTool?.tool.annotations?.readOnlyHint).toBe(true); - expect(snapshotTool?.tool.annotations?.idempotentHint).toBe(true); + expect(snapshotTool?.tool.annotations?.readOnlyHint).toBe(false); + expect(snapshotTool?.tool.annotations?.idempotentHint).toBe(false); expect(snapshotTool?.tool.annotations?.openWorldHint).toBe(true); const clickTool = server.tools.find(({ tool }) => tool.name === "preview_click"); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index f31c1576f5ae..f4da6e051a7c 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -818,7 +818,7 @@ const layerEnvironmentRegistration = toolkitRegistration( const layerProjectRegistration = toolkitRegistration(ProjectToolkit, ProjectHandlers.layer); -const layerAttachmentRegistration = toolkitRegistration( +export const layerAttachmentToolkit = toolkitRegistration( AttachmentToolkit, AttachmentHandlers.layer, ); @@ -854,7 +854,7 @@ export const layer = Layer.mergeAll( layerPreviewToolkit, layerOrchestratorToolkit, layerThreadToolkit, - layerAttachmentRegistration, + layerAttachmentToolkit, layerProjectRegistration, layerEnvironmentRegistration, layerPreviewControlsRegistration, diff --git a/apps/server/src/mcp/toolkits/attachment/handlers.ts b/apps/server/src/mcp/toolkits/attachment/handlers.ts index 9b9a6015d232..374aeb4543b4 100644 --- a/apps/server/src/mcp/toolkits/attachment/handlers.ts +++ b/apps/server/src/mcp/toolkits/attachment/handlers.ts @@ -1,5 +1,7 @@ import { type ChatAttachment, MessageId, OrchestratorMcpFailure } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as Upload from "../../../assets/AttachmentUpload.ts"; import * as Claims from "../../../orchestration-v2/AttachmentClaims.ts"; import * as ThreadMessageIntake from "../../../orchestration-v2/ThreadMessageIntake.ts"; @@ -27,53 +29,91 @@ export function resolveAttachmentReferences( }); } -export const layer = McpToolAccess.toLayer(AttachmentToolkit, { - t3_attachment_prepare_upload: McpToolAccess.writes((input) => - Upload.issueAttachmentUploadUrl(input.upload).pipe(Effect.mapError(unavailable)), - ), - t3_attachment_discard: McpToolAccess.writes((input) => - Upload.deletePendingAttachment(input.attachmentId).pipe(Effect.as({})), - ), - t3_thread_send_attachments: McpToolAccess.writesThreads( - (input) => [input.threadId], - (input) => - Effect.gen(function* () { - const { caller, projection } = yield* readThread(input.threadId, ["messages"]); - if (projection.thread.archivedAt !== null) - return yield* new OrchestratorMcpFailure({ - code: "invalid_request", - message: "Unarchive the target thread before sending attachments.", - }); - const attachments = yield* resolveAttachmentReferences( - input.attachments, - projection.messages.flatMap((message) => message.attachments), - ); - const commandId = yield* newCommandId(); - const messageId = MessageId.make(commandId); - const result = yield* ThreadMessageIntake.sendToThread({ - projectId: projection.thread.projectId, - threadId: projection.thread.id, - commandId, - messageId, - ...(caller === undefined ? {} : { senderThreadId: caller.id }), - text: input.message ?? "", - attachments, - mode: "auto", - createdBy: "agent", - creationSource: "mcp", - }).pipe( - Effect.mapError((error) => - error._tag === "AttachmentClaimError" - ? new OrchestratorMcpFailure({ code: "orchestration_error", message: error.message }) - : unavailable(), - ), - ); - return { - threadId: projection.thread.id, - messageId, - runId: result.run.id, - status: result.run.status, - }; - }), - ), -}); +/** Pending uploads outlive this record only until the attachment sweep removes them. */ +const UPLOAD_OWNER_TTL_MS = 24 * 60 * 60 * 1000; + +/** A thread owns its uploads across provider sessions; an outside client per MCP session. */ +const uploadOwner = McpInvocationContext.McpInvocationContext.pipe( + Effect.map((scope) => scope.thread?.threadId ?? scope.requestNamespace), +); + +export const layer = McpToolAccess.toLayer( + AttachmentToolkit, + Effect.sync(() => { + // Which caller prepared each pending upload, so only that caller can discard it. + const uploadOwners = new Map(); + return { + t3_attachment_prepare_upload: McpToolAccess.writes((input) => + Effect.gen(function* () { + const result = yield* Upload.issueAttachmentUploadUrl(input.upload).pipe( + Effect.mapError(unavailable), + ); + const now = yield* Clock.currentTimeMillis; + for (const [id, entry] of uploadOwners) { + if (now - entry.issuedAt > UPLOAD_OWNER_TTL_MS) uploadOwners.delete(id); + } + uploadOwners.set(result.attachmentId, { owner: yield* uploadOwner, issuedAt: now }); + return result; + }), + ), + t3_attachment_discard: McpToolAccess.writes((input) => + Effect.gen(function* () { + if (uploadOwners.get(input.attachmentId)?.owner !== (yield* uploadOwner)) { + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: "Only the caller that prepared a pending upload can discard it.", + }); + } + yield* Upload.deletePendingAttachment(input.attachmentId); + uploadOwners.delete(input.attachmentId); + return {}; + }), + ), + t3_thread_send_attachments: McpToolAccess.writesThreads( + (input) => [input.threadId], + (input) => + Effect.gen(function* () { + const { caller, projection } = yield* readThread(input.threadId, ["messages"]); + if (projection.thread.archivedAt !== null) + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: "Unarchive the target thread before sending attachments.", + }); + const attachments = yield* resolveAttachmentReferences( + input.attachments, + projection.messages.flatMap((message) => message.attachments), + ); + const commandId = yield* newCommandId(); + const messageId = MessageId.make(commandId); + const result = yield* ThreadMessageIntake.sendToThread({ + projectId: projection.thread.projectId, + threadId: projection.thread.id, + commandId, + messageId, + ...(caller === undefined ? {} : { senderThreadId: caller.id }), + text: input.message ?? "", + attachments, + mode: "auto", + createdBy: "agent", + creationSource: "mcp", + }).pipe( + Effect.mapError((error) => + error._tag === "AttachmentClaimError" + ? new OrchestratorMcpFailure({ + code: "orchestration_error", + message: error.message, + }) + : unavailable(), + ), + ); + return { + threadId: projection.thread.id, + messageId, + runId: result.run.id, + status: result.run.status, + }; + }), + ), + }; + }), +); diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index 634133b62e9e..5cf1953ba827 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -26,6 +26,7 @@ import { OrchestratorProjectionError, } from "../../orchestration-v2/Orchestrator.ts"; +import * as ServerSecretStore from "../../auth/ServerSecretStore.ts"; import * as ServerConfig from "../../config.ts"; import * as ProviderAdapterRegistry from "../../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; @@ -696,3 +697,52 @@ it.effect("a caller cannot interrupt a thread that runs above its own modes", () ), ), ); + +it.effect("only the caller that prepared a pending upload can discard it", () => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const call = ( + name: string, + args: Record, + invocation: McpInvocationContext.McpInvocationScope, + ) => + server + .callTool({ name, arguments: args }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + const prepared = yield* call( + "t3_attachment_prepare_upload", + { upload: { name: "shot.png", mimeType: "image/png", sizeBytes: 4 } }, + scope, + ); + const { attachmentId } = prepared.structuredContent as { readonly attachmentId: string }; + const otherThread = { + ...scope, + requestNamespace: "other-session", + thread: { ...scope.thread!, threadId: ThreadId.make("other-thread") }, + }; + + const refused = yield* call("t3_attachment_discard", { attachmentId }, otherThread); + expect(declaredFailure(refused)).toMatchObject({ code: "invalid_request" }); + + // A new provider session of the same thread still owns the upload. + const discarded = yield* call( + "t3_attachment_discard", + { attachmentId }, + { ...scope, requestNamespace: "mcp-core-session-2" }, + ); + expect(discarded.isError).toBe(false); + }).pipe( + Effect.provide( + McpHttpServer.layerAttachmentToolkit.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provide(McpToolAccessTestkit.liveThreadsLayer), + Layer.provide(ServerSecretStore.layer), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-attachment-" })), + Layer.provide(NodeServices.layer), + ), + ), + ), +); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index bf697229718c..1543bbc87100 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -211,15 +211,20 @@ const handlers = { const scope = yield* requireDeviceAccess; const devices = yield* DeviceService.DeviceService; const sessions = yield* devices.sessionsForThread(scope.thread.threadId); - const target = - input.deviceId !== undefined - ? { hostId: input.hostId ?? LOCAL_DEVICE_HOST_ID, deviceId: input.deviceId } - : sessions - .filter((session) => input.hostId === undefined || session.hostId === input.hostId) - .at(-1); + // Only devices this thread opened: another thread's device is not this agent's to watch. + const hostId = + input.deviceId === undefined ? input.hostId : (input.hostId ?? LOCAL_DEVICE_HOST_ID); + const target = sessions.findLast( + (session) => + (hostId === undefined || session.hostId === hostId) && + (input.deviceId === undefined || session.deviceId === input.deviceId), + ); if (!target) { return yield* new DeviceToolUnavailableError({ - reason: "No device is open in this thread. Call device_open first.", + reason: + input.deviceId === undefined + ? "No device is open in this thread. Call device_open first." + : `Device ${input.deviceId} on host ${hostId} is not open in this thread. Call device_open first.`, }); } const shot = yield* devices.screenshot(target); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index b35216fe4878..b9b966c760e3 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -139,7 +139,8 @@ const PreviewSetAppearanceTool = safeBrowserTool( .annotate(Tool.Idempotent, true), ); -export const PreviewSnapshotTool = readonlyBrowserTool( +// Not read-only: save=true writes the screenshot to disk. +export const PreviewSnapshotTool = safeBrowserTool( Tool.make("preview_snapshot", { description: "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and screenshot dimensions. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", From b41eca887dbe0865ec975794866bd48b7020b34a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 7 Oct 2026 16:03:48 -0700 Subject: [PATCH 5/7] feat(preview): agents read the user's browser tabs and open tabs under a chosen profile Agents could only see and drive the tabs their own session opened, and preview_open always used the built-in Default profile, so "ask my agent about the page I'm on" did not work. - preview_status lists every server tab in the thread with its owner (human / agent / unclaimed), ownedByCaller, visibility and profile. - Reads (status, snapshot, evaluate, wait_for) work on any tab in the thread, without queueing behind a human's input. With no tabId they fall back to the user's visible or most recently used tab when the agent has none of its own. - Actions work on a tab no agent opened while no human controls it, and are refused with humanControl while one does. Another agent session's tabs stay read-only. preview_open no longer implicitly reuses a tab the session did not open, and only the opener can close a tab. - Clients report their browser profiles and default (preview.reportProfiles); the server keeps the latest report in memory. preview_open takes a profileId (id or case-insensitive name), defaults to the reported default, and lists valid profiles on an unknown one. The profile reaches headless tabs' storage and the desktop's partition via the snapshot. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/auth/RpcAuthorization.test.ts | 1 + apps/server/src/auth/RpcAuthorization.ts | 1 + apps/server/src/mcp/toolkits/preview/tools.ts | 10 +- .../src/observability/RpcInstrumentation.ts | 1 + apps/server/src/preview/ServerBrowser.test.ts | 192 +++++++++++++++++- apps/server/src/preview/ServerBrowser.ts | 167 +++++++++++++-- .../src/preview/ServerBrowserStream.test.ts | 5 + .../server/src/preview/SessionControl.test.ts | 12 ++ apps/server/src/preview/SessionControl.ts | 27 ++- .../provider/T3OrchestrationInstructions.ts | 2 + apps/server/src/ws.ts | 1 + apps/web/src/AppRoot.tsx | 2 + .../src/browser/BrowserProfileReporter.tsx | 40 ++++ apps/web/src/browser/ElectronBrowserHost.tsx | 10 +- packages/client-runtime/src/state/preview.ts | 5 + packages/contracts/src/preview.ts | 10 +- packages/contracts/src/previewAutomation.ts | 38 +++- packages/contracts/src/rpc.ts | 8 + 18 files changed, 488 insertions(+), 44 deletions(-) create mode 100644 apps/web/src/browser/BrowserProfileReporter.tsx diff --git a/apps/server/src/auth/RpcAuthorization.test.ts b/apps/server/src/auth/RpcAuthorization.test.ts index 97a8f8dc0424..0359b07b3941 100644 --- a/apps/server/src/auth/RpcAuthorization.test.ts +++ b/apps/server/src/auth/RpcAuthorization.test.ts @@ -152,6 +152,7 @@ describe("RPC authorization scopes", () => { WS_METHODS.previewReportStatus, WS_METHODS.previewAdjust, WS_METHODS.previewClearProfile, + WS_METHODS.previewReportProfiles, ]) { expect(requiredScopeForRpcMethod(method)).toBe(AuthPreviewOperateScope); } diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index d434ca86a3a5..6e418e2eca35 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -187,6 +187,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.previewClose]: AuthPreviewOperateScope, [WS_METHODS.previewList]: AuthOrchestrationReadScope, [WS_METHODS.previewClearProfile]: AuthPreviewOperateScope, + [WS_METHODS.previewReportProfiles]: AuthPreviewOperateScope, [WS_METHODS.previewReportStatus]: AuthPreviewOperateScope, [WS_METHODS.subscribePreviewEvents]: AuthOrchestrationReadScope, [WS_METHODS.subscribeDiscoveredLocalServers]: AuthOrchestrationReadScope, diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index b9b966c760e3..850c54719323 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -64,7 +64,7 @@ const readonlyBrowserTool = (tool: T): T => const PreviewStatusTool = Tool.make("preview_status", { description: - "Report whether a collaborative browser tab is automation-capable, including its control owner, pending dialog, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.", + "Report whether a collaborative browser tab is automation-capable, including its control owner, pending dialog, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Server hosts also list every tab in the thread (tabs) with its owner, including tabs the user opened, and the browser profiles preview_open accepts. You can read any listed tab; act only on your own tabs, or on an unclaimed tab while no human controls it.", parameters: PreviewAutomationTabTargetInput, success: PreviewAutomationStatus, failure: PreviewToolFailure, @@ -78,7 +78,7 @@ const PreviewStatusTool = Tool.make("preview_status", { const PreviewOpenTool = browserTool( Tool.make("preview_open", { description: - "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Parallel subagents sharing a provider session must each open with reuseExistingTab=false and pass their returned tabId on every call. Server tabs use isolated storage and cannot be operated by a different agent session.", + "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Parallel subagents sharing a provider session must each open with reuseExistingTab=false and pass their returned tabId on every call. Pass profileId (an id or name from preview_status profiles) to open under a browser profile and its saved logins; omit it for the user's default profile. Another agent session's tabs can be read but not operated.", parameters: PreviewAutomationOpenInput, success: PreviewAutomationStatus, failure: PreviewToolFailure, @@ -91,7 +91,7 @@ const PreviewOpenTool = browserTool( const PreviewDialogTool = browserTool( Tool.make("preview_dialog", { description: - "Accept or dismiss the server browser dialog reported by preview_status. For a prompt, supply promptText when accepting. Requires this agent to own the tab. Desktop hosts may not support this operation.", + "Accept or dismiss the server browser dialog reported by preview_status. For a prompt, supply promptText when accepting. Requires this agent to own the tab, or the tab to be unclaimed with no human in control. Desktop hosts may not support this operation.", parameters: PreviewAutomationDialogInput, success: PreviewAutomationStatus, failure: PreviewToolFailure, @@ -143,7 +143,7 @@ const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = safeBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and screenshot dimensions. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab, including one the user opened (see preview_status tabs), even while they control it; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Returns page state, semantic elements, diagnostics, action history, and screenshot dimensions. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( @@ -271,7 +271,7 @@ export const PreviewEvaluateResult = Schema.Struct({ const PreviewEvaluateTool = browserTool( Tool.make("preview_evaluate", { description: - "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.", + "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Works on any tab in the thread, including the user's while they control it, so use it there only to read. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.", parameters: PreviewAutomationEvaluateInput, success: PreviewEvaluateResult, failure: PreviewToolFailure, diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index d01c7812bc1a..c8837d8619c9 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -181,6 +181,7 @@ const RPC_AGGREGATES = { [WS_METHODS.previewClose]: "preview", [WS_METHODS.previewList]: "preview", [WS_METHODS.previewClearProfile]: "preview", + [WS_METHODS.previewReportProfiles]: "preview", [WS_METHODS.previewReportStatus]: "preview", [WS_METHODS.subscribePreviewEvents]: "preview", [WS_METHODS.subscribeDiscoveredLocalServers]: "preview", diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index dce0c23b96cf..39bf49ad579f 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -36,7 +36,8 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ constructor(options: { onContextClose?: (context: BrowserContext) => void }) { this.onClose = options.onContextClose; } - async contextFor() { + async contextFor(profileId: string, isolationKey?: string) { + contextRequests.push({ profileId, isolated: isolationKey !== undefined }); if (contextFailure) throw contextFailure; await contextGate?.promise; const context = makeContext(this.onClose); @@ -147,6 +148,8 @@ function makeContext(onClose?: (context: BrowserContext) => void) { } const contexts: ReturnType[] = []; +/** The profile and isolation each headless tab asked its context for. */ +const contextRequests: Array<{ profileId: string; isolated: boolean }> = []; let contextGate: PromiseWithResolvers | null = null; type ClipboardBinding = (source: { page: unknown }, text: unknown) => void; let clipboardBinding: ClipboardBinding | null = null; @@ -253,6 +256,7 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({ beforeEach(() => { contexts.length = 0; + contextRequests.length = 0; contextGate = null; contextFailure = null; desktopTabs.clear(); @@ -367,18 +371,27 @@ it.live("enforces provider ownership and explicit targets when a session has mul .invoke({ scope: asSession("agent-b"), tabId, - operation: "evaluate", - input: { expression: "foreign()" }, + operation: "navigate", + input: { url: "http://localhost:5173/foreign" }, }) .pipe(Effect.flip); expect(foreign).toMatchObject({ _tag: "PreviewAutomationControlInterruptedError", reason: "agentMismatch", }); - expect(contexts[0]!.sessions[0]!.send).not.toHaveBeenCalledWith( - "Runtime.evaluate", + expect(contexts[0]!.page.goto).not.toHaveBeenCalledWith( + "http://localhost:5173/foreign", expect.anything(), ); + // Another session may still read the tab. + expect( + yield* broker.invoke({ + scope: asSession("agent-b"), + tabId, + operation: "evaluate", + input: { expression: "read()" }, + }), + ).toBe("evaluated"); yield* broker.invoke({ scope, operation: "open", @@ -1090,3 +1103,172 @@ it.live("a desktop page the desktop takes back reconnects instead of closing", ( }), ).pipe(Effect.provide(layer)), ); + +it.live("agents read a human's tab with no arguments and act on it only while nobody drives", () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + const opened = yield* manager.open({ + threadId: scope.thread.threadId, + url: "http://localhost:5173/mine", + runtime: "server", + }); + const tabId = PreviewTabId.make(opened.tabId); + // The user opened and is looking at the tab; attaching takes control. + const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); + const status = yield* broker.invoke({ + scope, + operation: "status", + input: {}, + }); + expect(status).toMatchObject({ + tabId, + control: { owner: "human", ownedByCaller: false }, + }); + expect(status.tabs).toEqual([ + expect.objectContaining({ tabId, owner: "human", ownedByCaller: false, visible: true }), + ]); + // Reads work while the user drives. + expect( + yield* broker.invoke({ scope, operation: "evaluate", input: { expression: "read()" } }), + ).toBe("evaluated"); + yield* broker.invoke({ + scope, + tabId, + operation: "snapshot", + input: {}, + }); + // Acting is refused while the user controls the tab. + const refused = yield* broker + .invoke({ + scope, + tabId, + operation: "navigate", + input: { url: "http://localhost:5173/agent" }, + }) + .pipe(Effect.flip); + expect(refused).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "humanControl", + }); + // Once they let go, the tab is unclaimed and the agent may act on it. + yield* viewer.input({ type: "releaseControl" }); + const released = yield* broker.invoke({ + scope, + tabId, + operation: "status", + input: {}, + }); + expect(released.control).toMatchObject({ owner: "unclaimed", ownedByCaller: false }); + yield* broker.invoke({ + scope, + tabId, + operation: "navigate", + input: { url: "http://localhost:5173/agent" }, + }); + expect(contexts[0]!.page.goto).toHaveBeenCalledWith( + "http://localhost:5173/agent", + expect.anything(), + ); + // Taking control back refuses the agent again. + yield* viewer.input({ type: "takeControl" }); + const retaken = yield* broker + .invoke({ scope, tabId, operation: "press", input: { key: "Enter" } }) + .pipe(Effect.flip); + expect(retaken).toMatchObject({ reason: "humanControl" }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a session's own tab stays its default while other sessions' tabs are listed", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const other = yield* broker.invoke({ + scope: asSession("agent-b"), + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + const status = yield* broker.invoke({ + scope, + operation: "status", + input: {}, + }); + expect(status.tabId).toBe(tabId); + expect(status.tabs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ tabId, owner: "agent", ownedByCaller: true }), + expect.objectContaining({ tabId: other.tabId, owner: "agent", ownedByCaller: false }), + ]), + ); + }), + ).pipe(Effect.provide(layer)), +); + +it.live( + "preview_open picks a reported profile by id or name and defaults to the reported one", + () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + yield* browser.reportProfiles({ + profiles: [{ id: "profile-work", name: "Work", kind: "persistent" }], + defaultProfileId: "profile-work", + }); + const openWith = (profileId?: string) => + broker.invoke({ + scope, + operation: "open", + input: { + reuseExistingTab: false, + show: false, + ...(profileId === undefined ? {} : { profileId }), + }, + }); + const byDefault = yield* openWith(); + const byName = yield* openWith("WORK"); + const byId = yield* openWith("incognito"); + const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); + const profileOf = (tabId: string | null) => + sessions.find((session) => session.tabId === tabId)?.profileId; + expect(profileOf(byDefault.tabId)).toBe("profile-work"); + expect(profileOf(byName.tabId)).toBe("profile-work"); + expect(profileOf(byId.tabId)).toBe("incognito"); + // The chosen profile reaches the headless tab's storage. + expect(contextRequests).toEqual([ + { profileId: "profile-work", isolated: false }, + { profileId: "profile-work", isolated: false }, + { profileId: "incognito", isolated: true }, + ]); + expect(byDefault).toMatchObject({ + defaultProfileId: "profile-work", + profiles: [ + { id: "default", name: "Default" }, + { id: "incognito", name: "Incognito", incognito: true }, + { id: "profile-work", name: "Work" }, + ], + }); + const unknown = yield* openWith("Personal").pipe(Effect.flip); + expect(unknown).toMatchObject({ + _tag: "PreviewAutomationExecutionError", + reason: + 'No browser profile is named "Personal". Use one of: Default (id default), Incognito (id incognito), Work (id profile-work).', + }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("an agent tab keeps throwaway storage when no client reported profiles", () => + Effect.scoped( + Effect.gen(function* () { + yield* ready; + expect(contextRequests).toEqual([{ profileId: "default", isolated: true }]); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 42dc916abb61..7ac3291dbc1e 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -2,8 +2,14 @@ // Screencasts ignore emulated device scale; real 2x keeps captures sharp. // --disable-gpu uses cheaper software compositing while preserving SwiftShader WebGL. import { + BUILT_IN_BROWSER_PROFILES, + DEFAULT_BROWSER_PROFILE_ID, FILL_PREVIEW_VIEWPORT, + findBrowserProfile, INCOGNITO_BROWSER_PROFILE_ID, + resolveBrowserProfiles, + type BrowserProfile, + type PreviewReportProfilesInput, PREVIEW_AUTOMATION_SERVER_OPERATIONS, PreviewViewportSetting as PreviewViewportSettingSchema, PROVIDER_SEND_TURN_MAX_FILE_BYTES, @@ -237,6 +243,11 @@ export class ServerBrowser extends Context.Service< }) => Effect.Effect>; /** Deletes a human profile's server-side storage, closing its open tabs first. */ readonly clearProfile: (profileId: string) => Effect.Effect; + /** + * Records a client's browser profiles for agents' preview_open. In memory: + * clients report again on connect and whenever the list changes. + */ + readonly reportProfiles: (input: PreviewReportProfilesInput) => Effect.Effect; } >()("t3/preview/ServerBrowser") {} @@ -329,6 +340,14 @@ interface ServerDownload { } /** One agent session and the whole server; each tab holds a renderer process. */ +/** Operations an agent may run on any tab in its thread, even while a human drives it. */ +const READ_OPERATIONS: ReadonlySet = new Set([ + "status", + "snapshot", + "evaluate", + "waitFor", +]); + const AGENT_TAB_LIMIT = 8; const SERVER_TAB_LIMIT = 32; /** Agent tabs nobody watches close after this long without an agent request. */ @@ -706,10 +725,11 @@ const make = Effect.gen(function* () { adopted === undefined && (await desktopRenders(snapshot)) ? await connectDesktop(snapshot) : null; + // An agent tab without a profile (no client reported one) keeps throwaway storage. const isolatedContext = adopted === undefined && desktop === null && - (snapshot.automationOwner !== undefined || + ((snapshot.automationOwner !== undefined && snapshot.profileId === undefined) || snapshot.profileId === INCOGNITO_BROWSER_PROFILE_ID); const context = adopted?.page.context() ?? @@ -1091,10 +1111,75 @@ const make = Effect.gen(function* () { } }; - const latestThreadTab = (threadId: string, agentSessionId?: string) => - [...tabs.values()] - .filter((tab) => tab.threadId === threadId && tab.control.agentId === agentSessionId) - .sort((left, right) => right.createdAt - left.createdAt)[0]; + /** + * The tab a request without a tabId means: the caller's newest tab, else the + * thread's tab the user is looking at or used last, so "this page" works. + */ + const latestThreadTab = (threadId: string, agentSessionId?: string) => { + const threadTabs = [...tabs.values()].filter((tab) => tab.threadId === threadId); + return ( + threadTabs + .filter((tab) => tab.control.agentId === agentSessionId) + .sort((left, right) => right.createdAt - left.createdAt)[0] ?? + threadTabs.sort( + (left, right) => + Number(right.viewers.size > 0) - Number(left.viewers.size > 0) || + right.usedAt - left.usedAt, + )[0] + ); + }; + + const tabOwner = (tab: ServerTab) => + tab.control.controller !== null + ? ("human" as const) + : tab.control.agentId !== null + ? ("agent" as const) + : ("unclaimed" as const); + + /** The latest profile list a client reported; agents choose from it. */ + let reportedProfiles: { + readonly profiles: ReadonlyArray; + readonly defaultProfileId: string; + } | null = null; + + const reportProfiles: ServerBrowser["Service"]["reportProfiles"] = (input) => + Effect.sync(() => { + const profiles = resolveBrowserProfiles(input.profiles); + reportedProfiles = { + profiles, + defaultProfileId: + findBrowserProfile(profiles, input.defaultProfileId)?.id ?? DEFAULT_BROWSER_PROFILE_ID, + }; + }); + + const profileStatus = () => { + const profiles = reportedProfiles?.profiles ?? BUILT_IN_BROWSER_PROFILES; + return { + profiles: profiles.map((profile) => ({ + id: profile.id, + name: profile.name, + ...(profile.kind === "incognito" ? { incognito: true } : {}), + })), + ...(reportedProfiles ? { defaultProfileId: reportedProfiles.defaultProfileId } : {}), + }; + }; + + /** Matches an id, then a case-insensitive name. Omitted means the reported default. */ + const resolveOpenProfile = (requested: string | undefined): string | undefined => { + if (requested === undefined) return reportedProfiles?.defaultProfileId; + const profiles = reportedProfiles?.profiles ?? BUILT_IN_BROWSER_PROFILES; + const wanted = requested.toLowerCase(); + const match = + findBrowserProfile(profiles, requested) ?? + profiles.find((profile) => profile.name.toLowerCase() === wanted); + if (match) return match.id; + throw new ServerBrowserPage.ServerBrowserOperationError( + "PreviewAutomationUnknownProfileError", + `No browser profile is named "${requested}". Use one of: ${profiles + .map((profile) => `${profile.name} (id ${profile.id})`) + .join(", ")}.`, + ); + }; const statusWithTitle = async ( tab: ServerTab | undefined, @@ -1108,6 +1193,7 @@ const make = Effect.gen(function* () { url: null, title: null, loading: false, + ...profileStatus(), }; } const url = tab.page.url(); @@ -1120,12 +1206,7 @@ const make = Effect.gen(function* () { title: null, loading: tab.loading, control: { - owner: - tab.control.controller !== null - ? ("human" as const) - : tab.control.agentId !== null - ? ("agent" as const) - : ("unclaimed" as const), + owner: tabOwner(tab), ownedByCaller: tab.control.agentId === agentSessionId, generation: tab.control.generation, }, @@ -1136,15 +1217,17 @@ const make = Effect.gen(function* () { viewportSetting: tab.setting, ...(viewport ? { viewport } : {}), tabs: [...tabs.values()] - .filter( - (candidate) => - candidate.threadId === tab.threadId && candidate.control.agentId === agentSessionId, - ) + .filter((candidate) => candidate.threadId === tab.threadId) .map((candidate) => ({ tabId: candidate.tabId, url: candidate.page.url() === "about:blank" ? null : candidate.page.url(), ...(candidate.openerTabId === undefined ? {} : { openerTabId: candidate.openerTabId }), + owner: tabOwner(candidate), + ownedByCaller: candidate.control.agentId === agentSessionId, + visible: candidate.viewers.size > 0, + ...(candidate.profileId === undefined ? {} : { profileId: candidate.profileId }), })), + ...profileStatus(), downloads: tab.downloads.map(({ fileName, path, sizeBytes, url, completedAt }) => ({ fileName, path, @@ -1440,19 +1523,26 @@ const make = Effect.gen(function* () { "No server preview tab is open for this thread. Call preview_open first.", ); } - if (tab.control.agentId !== request.agentSessionId) + // Any tab in the thread can be read; acting is checked by its control. + if (!READ_OPERATIONS.has(request.operation) && !tab.control.agentMayAct(request.agentSessionId)) throw new BrowserControlInterrupted( - "This tab belongs to another agent session or a human. Open your own tab.", + "This tab belongs to another agent session. You can read it, but open your own tab to act.", "agentMismatch", ); return tab; }; - /** Any request, even a failed one, keeps the agent's tabs on the thread from idling out. */ + /** + * Any request, even a failed one, keeps the agent's tabs on the thread from + * idling out. The tab it targeted becomes the thread's most recently used. + */ const markUsed = (request: PreviewAutomationRequest) => { const now = Date.now(); for (const tab of tabs.values()) { - if (tab.threadId === request.threadId && tab.control.agentId === request.agentSessionId) + if ( + tab.threadId === request.threadId && + (tab.control.agentId === request.agentSessionId || tab.tabId === request.tabId) + ) tab.usedAt = now; } }; @@ -1488,7 +1578,7 @@ const make = Effect.gen(function* () { "tabRequired", ); // A tab still launching exists only as a session, so resolve it like a viewer would. - const existing = + const found = reuse && request.tabId !== undefined ? await Effect.runPromise( findTab(request.threadId, request.tabId).pipe( @@ -1498,7 +1588,14 @@ const make = Effect.gen(function* () { ), ) : undefined; + // Only an explicit tabId reuses a tab this session did not open; a tab + // it last read (such as the user's) is not taken over implicitly. + const existing = + found && (request.tabIdExplicit || found.control.agentId === request.agentSessionId) + ? found + : undefined; const navigationTimeout = Math.min(request.timeoutMs, NAVIGATION_TIMEOUT_MS); + const profileId = existing ? undefined : resolveOpenProfile(open.profileId); if (!existing) { closeIdleAgentTabs(); assertTabCapacity(request.agentSessionId); @@ -1510,6 +1607,7 @@ const make = Effect.gen(function* () { manager.open({ threadId: request.threadId, ...(url ? { url } : {}), + ...(profileId === undefined ? {} : { profileId }), runtime: "server", reveal: false, automationOwner: request.agentSessionId, @@ -1567,7 +1665,7 @@ const make = Effect.gen(function* () { const recordings = [...tabs.values()].filter( (candidate) => candidate.threadId === request.threadId && - candidate.control.agentId === request.agentSessionId && + candidate.control.agentMayAct(request.agentSessionId ?? "") && (candidate.recording || candidate.recordingStart), ); const targetTabId = @@ -1587,6 +1685,11 @@ const make = Effect.gen(function* () { const tab = await requireTab(request); // Closing must unblock an action waiting on a dialog, without queueing behind it. if (request.operation === "close") { + if (tab.control.agentId !== request.agentSessionId) + throw new BrowserControlInterrupted( + "Only the agent session that opened this tab can close it.", + "agentMismatch", + ); if (tab.control.controller !== null) throw new BrowserControlInterrupted("A human controls this tab.", "humanControl"); void tab.control.close().catch(constVoid); @@ -1602,7 +1705,26 @@ const make = Effect.gen(function* () { await resolveDialog(tab, input as PreviewAutomationDialogInput); return statusWithTitle(tab, request.agentSessionId); } - return tab.control.agent(request.agentSessionId!, async () => { + const agentSessionId = request.agentSessionId!; + // A tab this session did not open, while it cannot act there (another + // agent's, or the user's while they drive): read it in place, without + // queueing behind the human's input. Its own tabs keep the takeover + // contract: a human taking control interrupts every agent call. + if ( + READ_OPERATIONS.has(request.operation) && + tab.control.agentId !== agentSessionId && + !tab.control.agentCanActNow(agentSessionId) + ) { + return tab.control.observe(async () => { + if (tab.dialog) + throw new BrowserControlInterrupted( + "A browser dialog is pending. Read preview_status.", + "dialogPending", + ); + return executeTabOperation(tab, request); + }); + } + return tab.control.agent(agentSessionId, async () => { if (tab.dialog) throw new BrowserControlInterrupted( "A browser dialog is pending. Read preview_status and use preview_dialog first.", @@ -2194,6 +2316,7 @@ const make = Effect.gen(function* () { return ServerBrowser.of({ attachViewer, clearProfile, + reportProfiles, openDownload, answerFileChooser, }); diff --git a/apps/server/src/preview/ServerBrowserStream.test.ts b/apps/server/src/preview/ServerBrowserStream.test.ts index e3366178b668..0654ea62a2b8 100644 --- a/apps/server/src/preview/ServerBrowserStream.test.ts +++ b/apps/server/src/preview/ServerBrowserStream.test.ts @@ -87,6 +87,7 @@ it.effect.each([ }); const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: () => Effect.succeedNone, answerFileChooser: () => Effect.succeed(false), attachViewer: (input) => @@ -163,6 +164,7 @@ it.effect.each([ let attachments = 0; const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: () => Effect.succeedNone, answerFileChooser: () => Effect.succeed(false), attachViewer: () => { @@ -203,6 +205,7 @@ it.effect("serves a tab's download only to an authorized session", () => const requests: Array = []; const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: (input) => Effect.sync(() => { requests.push(input); @@ -250,6 +253,7 @@ it.effect("passes uploaded files to the page's open picker and needs operate sco const answers: Array<{ chooserId: string; files: Array<{ name: string; text: string }> }> = []; const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: () => Effect.succeedNone, answerFileChooser: (input) => Effect.sync(() => { @@ -317,6 +321,7 @@ it.effect.each([ Effect.gen(function* () { const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: () => Effect.succeedNone, answerFileChooser: () => Effect.succeed(false), attachViewer: () => Effect.fail(new ServerBrowser.ServerBrowserLaunchError({ cause: error })), diff --git a/apps/server/src/preview/SessionControl.test.ts b/apps/server/src/preview/SessionControl.test.ts index 73d64cee1019..42eabc212ac0 100644 --- a/apps/server/src/preview/SessionControl.test.ts +++ b/apps/server/src/preview/SessionControl.test.ts @@ -166,4 +166,16 @@ describe("SessionControl", () => { ).rejects.toThrow("navigation failed"); await expect(control.agent("agent", async () => "recovered")).resolves.toBe("recovered"); }); + + it("lets any agent act on a tab no agent opened, but only while no human controls it", async () => { + const control = new SessionControl(null); + await expect(control.agent("agent-a", async () => "acted")).resolves.toBe("acted"); + await control.take("viewer-a"); + await expect(control.agent("agent-a", async () => "racing")).rejects.toMatchObject({ + reason: "humanControl", + }); + await expect(control.observe(async () => "read")).resolves.toBe("read"); + await control.release("viewer-a"); + await expect(control.agent("agent-b", async () => "acted")).resolves.toBe("acted"); + }); }); diff --git a/apps/server/src/preview/SessionControl.ts b/apps/server/src/preview/SessionControl.ts index a361cd1a467c..a930a5e115fc 100644 --- a/apps/server/src/preview/SessionControl.ts +++ b/apps/server/src/preview/SessionControl.ts @@ -73,8 +73,17 @@ export class SessionControl { }); } + /** Whether an agent may act here: its own tab, or a tab no agent opened. */ + agentMayAct(agentId: string) { + return this.agentId === null || this.agentId === agentId; + } + + /** + * An agent action. It may act on its own tab or on one a human opened, but + * never while a human controls the tab; taking control interrupts it. + */ agent(agentId: string, run: () => Promise) { - if (this.agentId !== agentId) + if (!this.agentMayAct(agentId)) return Promise.reject( new BrowserControlInterrupted("This tab belongs to another agent.", "agentMismatch"), ); @@ -82,7 +91,21 @@ export class SessionControl { return Promise.reject( new BrowserControlInterrupted("A human controls this tab.", "humanControl"), ); - return this.action(() => this.agentId === agentId && this.owner === null, run); + return this.action(() => this.agentMayAct(agentId) && this.owner === null, run); + } + + /** Whether this agent's action would run now rather than be refused. */ + agentCanActNow(agentId: string) { + return this.agentMayAct(agentId) && this.owner === null; + } + + /** + * A read that needs no control and does not queue, so an agent can look at + * a page while a human drives it without holding up the human's input. + */ + observe(run: () => Promise) { + this.assertOpen(); + return run(); } /** diff --git a/apps/server/src/provider/T3OrchestrationInstructions.ts b/apps/server/src/provider/T3OrchestrationInstructions.ts index 6412991f34e7..48998751409b 100644 --- a/apps/server/src/provider/T3OrchestrationInstructions.ts +++ b/apps/server/src/provider/T3OrchestrationInstructions.ts @@ -43,6 +43,8 @@ You are running inside T3 Code. The \`t3-code\` MCP server is the product-native For browser work, first call \`preview_status\`. If no automation-capable preview is attached, call \`preview_open\` before concluding that the browser is unavailable. Then use \`preview_navigate\`, \`preview_snapshot\`, and the focused interaction tools. Prefer snapshot-provided locators over coordinates. +\`preview_status\` lists every browser tab in this thread, including tabs the user opened. When the user asks about "this page" or a page they have open, read their tab: pass its \`tabId\` to \`preview_snapshot\`, \`preview_evaluate\`, or \`preview_wait_for\`, or omit \`tabId\` when you have no tab of your own. You may act on the user's tab only while its owner is \`unclaimed\`; while it is \`human\`, the user is driving, so do not act there. Read it, or open your own tab. To use a browser profile (a set of saved logins), pass \`profileId\` from \`preview_status\` profiles to \`preview_open\`. + Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Inspect a failed preview call and retry with corrected arguments when the error is actionable. Use another browser system when: - the T3 preview tools are absent, or \`preview_open\` returns an explicit unsupported/unavailable error; - the user asks for another browser, or invokes a skill or documented repository workflow that names one; follow that workflow and report any prerequisite it is missing; diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index af3fc819a4c7..7c131dba1330 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2884,6 +2884,7 @@ const layerWsRpc = ( [WS_METHODS.previewClose]: (input) => previewManager.close(input), [WS_METHODS.previewList]: (input) => previewManager.list(input), [WS_METHODS.previewClearProfile]: (input) => serverBrowser.clearProfile(input.profileId), + [WS_METHODS.previewReportProfiles]: (input) => serverBrowser.reportProfiles(input), [WS_METHODS.previewReportStatus]: (input) => previewManager.reportStatus(input), [WS_METHODS.subscribePreviewEvents]: (_input) => previewManager.events, [WS_METHODS.deviceConfigure]: (input) => deviceService.configure(input), diff --git a/apps/web/src/AppRoot.tsx b/apps/web/src/AppRoot.tsx index 443c260f5e86..1ec12cfdd712 100644 --- a/apps/web/src/AppRoot.tsx +++ b/apps/web/src/AppRoot.tsx @@ -1,5 +1,6 @@ import { RouterProvider } from "@tanstack/react-router"; +import { BrowserProfileReporter } from "./browser/BrowserProfileReporter"; import { ElectronBrowserHost } from "./browser/ElectronBrowserHost"; import { QuitHoldOverlay } from "./components/QuitHoldOverlay"; import { AppAtomRegistryProvider } from "./rpc/atomRegistry"; @@ -15,6 +16,7 @@ export function AppRoot({ router }: { readonly router: AppRouter }) { + ); diff --git a/apps/web/src/browser/BrowserProfileReporter.tsx b/apps/web/src/browser/BrowserProfileReporter.tsx new file mode 100644 index 000000000000..e148f79d72af --- /dev/null +++ b/apps/web/src/browser/BrowserProfileReporter.tsx @@ -0,0 +1,40 @@ +import { AuthPreviewOperateScope, type BrowserProfile } from "@t3tools/contracts"; +import { useEffect } from "react"; + +import { useClientSettings, useClientSettingsHydrated } from "~/hooks/useSettings"; +import { useServerConfigs } from "~/state/entities"; +import { useEnvironments } from "~/state/environments"; +import { previewEnvironment } from "~/state/preview"; +import { useEnvironmentsWithScope } from "~/state/session"; +import { useAtomCommand } from "~/state/use-atom-command"; + +const selectProfiles = (settings: { readonly browserProfiles: ReadonlyArray }) => + settings.browserProfiles; +const selectDefaultProfileId = (settings: { readonly browserDefaultProfileId: string }) => + settings.browserDefaultProfileId; + +/** + * Tells each environment that hosts browser tabs which profiles this client + * has, so agents can open tabs under them. Profiles live in client settings; + * the server keeps only the latest report in memory. A reconnect brings a new + * server config, which sends the report again. + */ +export function BrowserProfileReporter() { + const hydrated = useClientSettingsHydrated(); + const profiles = useClientSettings(selectProfiles); + const defaultProfileId = useClientSettings(selectDefaultProfileId); + const { environments } = useEnvironments(); + const serverConfigs = useServerConfigs(); + const operable = useEnvironmentsWithScope(environments, AuthPreviewOperateScope); + const report = useAtomCommand(previewEnvironment.reportProfiles, { reportFailure: false }); + + useEffect(() => { + if (!hydrated) return; + for (const [environmentId, config] of serverConfigs) { + if (!config.environment.capabilities.serverBrowser || !operable.has(environmentId)) continue; + void report({ environmentId, input: { profiles, defaultProfileId } }); + } + }, [defaultProfileId, hydrated, operable, profiles, report, serverConfigs]); + + return null; +} diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index 1c3be69ad779..629abc103077 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -136,11 +136,11 @@ function AuthorizedBrowserWebview(props: ComponentProps( label: "environment-data:preview:clear-profile", tag: WS_METHODS.previewClearProfile, }), + reportProfiles: createEnvironmentRpcCommand(runtime, { + label: "environment-data:preview:report-profiles", + tag: WS_METHODS.previewReportProfiles, + concurrency: { mode: "latest", key: ({ environmentId }) => environmentId }, + }), reportStatus: createEnvironmentRpcCommand(runtime, { label: "environment-data:preview:report-status", tag: WS_METHODS.previewReportStatus, diff --git a/packages/contracts/src/preview.ts b/packages/contracts/src/preview.ts index e3e964192471..de458cfc035e 100644 --- a/packages/contracts/src/preview.ts +++ b/packages/contracts/src/preview.ts @@ -10,7 +10,7 @@ */ import { Schema } from "effect"; import { NonNegativeInt, PositiveInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; -import { BrowserProfileId } from "./browserProfile.ts"; +import { BROWSER_PROFILE_MAX_COUNT, BrowserProfile, BrowserProfileId } from "./browserProfile.ts"; export const PREVIEW_URL_MAX_LENGTH = 2_048; export const CONFIGURED_LOCAL_SERVER_URLS_MAX_ITEMS = 32; @@ -286,6 +286,14 @@ export const PreviewCloseInput = Schema.Struct({ }); export type PreviewCloseInput = typeof PreviewCloseInput.Type; +/** A client's browser profiles, which agents choose from when they open a tab. */ +export const PreviewReportProfilesInput = Schema.Struct({ + /** The user's own profiles; the server adds the built-ins. */ + profiles: Schema.Array(BrowserProfile).check(Schema.isMaxLength(BROWSER_PROFILE_MAX_COUNT)), + defaultProfileId: BrowserProfileId, +}); +export type PreviewReportProfilesInput = typeof PreviewReportProfilesInput.Type; + export const PreviewClearProfileInput = Schema.Struct({ profileId: BrowserProfileId, }); diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index fa16511bd3e8..6634e73b6990 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -10,12 +10,15 @@ import { PreviewViewportSize, } from "./preview.ts"; import { ProviderInstanceId } from "./providerInstance.ts"; +import { BrowserProfileId } from "./browserProfile.ts"; const BoundedUrl = Schema.String.check(Schema.isTrimmed()) .check(Schema.isNonEmpty()) .check(Schema.isMaxLength(2048)); const URL_GUIDANCE = "Absolute http(s) URL or a schemeless host such as t3.chat or localhost:5173. Schemeless public hosts use https; loopback hosts use http."; +const PROFILE_GUIDANCE = + "Browser profile for a new tab, by id or name (case-insensitive) from preview_status profiles. Omit to use the user's default profile. A tab keeps the profile it opened with."; const OptionalTimeoutMs = Schema.optional( Schema.Int.check(Schema.isGreaterThan(0)) .check(Schema.isLessThanOrEqualTo(60_000)) @@ -61,17 +64,27 @@ const PreviewAutomationTabTargetFields = { tabId: Schema.optional( PreviewTabId.annotate({ description: - "Exact collaborative browser tab to target. Omit to use this agent session's current tab.", + "Exact collaborative browser tab to target, including a tab the user opened (see preview_status tabs). Omit to use this agent session's current tab, or the user's visible tab when this session has none.", }), ).annotate({ description: - "Exact collaborative browser tab to target. Omit to use this agent session's current tab.", + "Exact collaborative browser tab to target, including a tab the user opened (see preview_status tabs). Omit to use this agent session's current tab, or the user's visible tab when this session has none.", }), }; export const PreviewAutomationTabTargetInput = Schema.Struct(PreviewAutomationTabTargetFields); export type PreviewAutomationTabTargetInput = typeof PreviewAutomationTabTargetInput.Type; +/** `human`: a person controls the tab now; `agent`: an agent session opened it; `unclaimed`: neither. */ +export const PreviewAutomationTabOwner = Schema.Literals(["agent", "human", "unclaimed"]); + +export const PreviewAutomationProfile = Schema.Struct({ + id: BrowserProfileId, + name: Schema.String, + incognito: Schema.optional(Schema.Boolean), +}); +export type PreviewAutomationProfile = typeof PreviewAutomationProfile.Type; + export const PreviewAutomationStatus = Schema.Struct({ available: Schema.Boolean, visible: Schema.Boolean, @@ -81,7 +94,7 @@ export const PreviewAutomationStatus = Schema.Struct({ loading: Schema.Boolean, control: Schema.optional( Schema.Struct({ - owner: Schema.Literals(["agent", "human", "unclaimed"]), + owner: PreviewAutomationTabOwner, ownedByCaller: Schema.Boolean, generation: Schema.Number, }), @@ -115,16 +128,28 @@ export const PreviewAutomationStatus = Schema.Struct({ }), ), ), - /** Server hosts: every tab this agent session owns, including popups its pages opened. */ + /** + * Server hosts: every tab in the thread, including ones the user or another + * agent session opened. Any of them can be read; acting needs `ownedByCaller`, + * or an `unclaimed` tab while no human controls it. + */ tabs: Schema.optional( Schema.Array( Schema.Struct({ tabId: PreviewTabId, url: Schema.NullOr(Schema.String), openerTabId: Schema.optional(PreviewTabId), + owner: Schema.optional(PreviewAutomationTabOwner), + ownedByCaller: Schema.optional(Schema.Boolean), + visible: Schema.optional(Schema.Boolean), + profileId: Schema.optional(BrowserProfileId), }), ), ), + /** Server hosts: the browser profiles preview_open accepts, as the user's client reported them. */ + profiles: Schema.optional(Schema.Array(PreviewAutomationProfile)), + /** Server hosts: the profile preview_open uses when given none. */ + defaultProfileId: Schema.optional(BrowserProfileId), }); export type PreviewAutomationStatus = typeof PreviewAutomationStatus.Type; @@ -162,6 +187,11 @@ export const PreviewAutomationOpenInput = Schema.Struct({ "Reuse tabId when supplied, otherwise this agent session's current tab. Defaults to true; set false to create a new tab.", }), ), + profileId: Schema.optional( + TrimmedNonEmptyString.check(Schema.isMaxLength(64)).annotate({ + description: PROFILE_GUIDANCE, + }), + ).annotate({ description: PROFILE_GUIDANCE }), }) .check( Schema.makeFilter( diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 20645ac4755b..9eba75c978a7 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -251,6 +251,7 @@ import { PreviewListResult, PreviewClearProfileError, PreviewClearProfileInput, + PreviewReportProfilesInput, PreviewNavigateInput, PreviewOpenInput, PreviewRefreshInput, @@ -439,6 +440,7 @@ export const WS_METHODS = { previewClose: "preview.close", previewList: "preview.list", previewClearProfile: "preview.clearProfile", + previewReportProfiles: "preview.reportProfiles", previewReportStatus: "preview.reportStatus", // Device methods @@ -1470,6 +1472,11 @@ const WsPreviewClearProfileRpc = Rpc.make(WS_METHODS.previewClearProfile, { error: Schema.Union([PreviewClearProfileError, EnvironmentAuthorizationError]), }); +const WsPreviewReportProfilesRpc = Rpc.make(WS_METHODS.previewReportProfiles, { + payload: PreviewReportProfilesInput, + error: EnvironmentAuthorizationError, +}); + const WsPreviewReportStatusRpc = Rpc.make(WS_METHODS.previewReportStatus, { payload: PreviewReportStatusInput, error: Schema.Union([PreviewError, EnvironmentAuthorizationError]), @@ -1934,6 +1941,7 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewCloseRpc, WsPreviewListRpc, WsPreviewClearProfileRpc, + WsPreviewReportProfilesRpc, WsPreviewReportStatusRpc, WsSubscribePreviewEventsRpc, WsSubscribeDiscoveredLocalServersRpc, From ce243e150f77ad6e0f749c7c34868b951a3acbca Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 7 Oct 2026 16:18:16 -0700 Subject: [PATCH 6/7] fix(server): preview_evaluate needs control of the tab Page script can change anything, so evaluate is no longer a read an agent may run on a tab it cannot act on: the user's tab while they drive it, or another agent session's tab. Agents read those with preview_snapshot. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- apps/server/src/preview/ServerBrowser.test.ts | 32 +++++++++++++------ apps/server/src/preview/ServerBrowser.ts | 7 ++-- .../provider/T3OrchestrationInstructions.ts | 2 +- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 850c54719323..6e34a330025e 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -271,7 +271,7 @@ export const PreviewEvaluateResult = Schema.Struct({ const PreviewEvaluateTool = browserTool( Tool.make("preview_evaluate", { description: - "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Works on any tab in the thread, including the user's while they control it, so use it there only to read. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.", + "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Needs the same control as clicking: it works on your own tabs and on the user's tab only while nobody controls it. To read a tab the user is driving, use preview_snapshot. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.", parameters: PreviewAutomationEvaluateInput, success: PreviewEvaluateResult, failure: PreviewToolFailure, diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 39bf49ad579f..05793e4640ce 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -383,15 +383,25 @@ it.live("enforces provider ownership and explicit targets when a session has mul "http://localhost:5173/foreign", expect.anything(), ); - // Another session may still read the tab. - expect( - yield* broker.invoke({ + // Another session may still read the tab, but not run page script in it. + yield* broker.invoke({ + scope: asSession("agent-b"), + tabId, + operation: "snapshot", + input: {}, + }); + const foreignEvaluate = yield* broker + .invoke({ scope: asSession("agent-b"), tabId, operation: "evaluate", input: { expression: "read()" }, - }), - ).toBe("evaluated"); + }) + .pipe(Effect.flip); + expect(foreignEvaluate).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "agentMismatch", + }); yield* broker.invoke({ scope, operation: "open", @@ -1131,10 +1141,14 @@ it.live("agents read a human's tab with no arguments and act on it only while no expect(status.tabs).toEqual([ expect.objectContaining({ tabId, owner: "human", ownedByCaller: false, visible: true }), ]); - // Reads work while the user drives. - expect( - yield* broker.invoke({ scope, operation: "evaluate", input: { expression: "read()" } }), - ).toBe("evaluated"); + // Reads work while the user drives; page script does not, since it can change the page. + const evaluated = yield* broker + .invoke({ scope, operation: "evaluate", input: { expression: "read()" } }) + .pipe(Effect.flip); + expect(evaluated).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "humanControl", + }); yield* broker.invoke({ scope, tabId, diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 7ac3291dbc1e..5e170251b0f3 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -340,11 +340,14 @@ interface ServerDownload { } /** One agent session and the whole server; each tab holds a renderer process. */ -/** Operations an agent may run on any tab in its thread, even while a human drives it. */ +/** + * Operations an agent may run on any tab in its thread, even while a human + * drives it. Evaluate is not one: page script can change anything, so it needs + * the same control as clicking. + */ const READ_OPERATIONS: ReadonlySet = new Set([ "status", "snapshot", - "evaluate", "waitFor", ]); diff --git a/apps/server/src/provider/T3OrchestrationInstructions.ts b/apps/server/src/provider/T3OrchestrationInstructions.ts index 48998751409b..99596d4d8490 100644 --- a/apps/server/src/provider/T3OrchestrationInstructions.ts +++ b/apps/server/src/provider/T3OrchestrationInstructions.ts @@ -43,7 +43,7 @@ You are running inside T3 Code. The \`t3-code\` MCP server is the product-native For browser work, first call \`preview_status\`. If no automation-capable preview is attached, call \`preview_open\` before concluding that the browser is unavailable. Then use \`preview_navigate\`, \`preview_snapshot\`, and the focused interaction tools. Prefer snapshot-provided locators over coordinates. -\`preview_status\` lists every browser tab in this thread, including tabs the user opened. When the user asks about "this page" or a page they have open, read their tab: pass its \`tabId\` to \`preview_snapshot\`, \`preview_evaluate\`, or \`preview_wait_for\`, or omit \`tabId\` when you have no tab of your own. You may act on the user's tab only while its owner is \`unclaimed\`; while it is \`human\`, the user is driving, so do not act there. Read it, or open your own tab. To use a browser profile (a set of saved logins), pass \`profileId\` from \`preview_status\` profiles to \`preview_open\`. +\`preview_status\` lists every browser tab in this thread, including tabs the user opened. When the user asks about "this page" or a page they have open, read their tab: pass its \`tabId\` to \`preview_snapshot\` or \`preview_wait_for\`, or omit \`tabId\` when you have no tab of your own. You may act on the user's tab, including \`preview_evaluate\`, only while its owner is \`unclaimed\`; while it is \`human\`, the user is driving, so read it with \`preview_snapshot\` or open your own tab. To use a browser profile (a set of saved logins), pass \`profileId\` from \`preview_status\` profiles to \`preview_open\`. Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Inspect a failed preview call and retry with corrected arguments when the error is actionable. Use another browser system when: - the T3 preview tools are absent, or \`preview_open\` returns an explicit unsupported/unavailable error; From 2668af0a9387f510bdffd38739fb7611a52b29d1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 8 Oct 2026 17:02:16 -0700 Subject: [PATCH 7/7] fix(server): an agent keeps its pending upload until the file itself expires The ownership record expired 24 hours after the upload URL was issued, but the pending file's 24 hours start when the upload finishes, up to the URL's 10-minute lifetime later. A prepare in that gap dropped the record, so the owner could no longer discard a live upload. Co-Authored-By: Claude Opus 5.5 --- .../server/src/mcp/toolkits/attachment/handlers.ts | 14 +++++++++++--- apps/server/src/mcp/toolkits/core.test.ts | 10 ++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/apps/server/src/mcp/toolkits/attachment/handlers.ts b/apps/server/src/mcp/toolkits/attachment/handlers.ts index 374aeb4543b4..2236d8f305de 100644 --- a/apps/server/src/mcp/toolkits/attachment/handlers.ts +++ b/apps/server/src/mcp/toolkits/attachment/handlers.ts @@ -1,4 +1,9 @@ -import { type ChatAttachment, MessageId, OrchestratorMcpFailure } from "@t3tools/contracts"; +import { + ATTACHMENT_UPLOAD_URL_TTL_MS, + type ChatAttachment, + MessageId, + OrchestratorMcpFailure, +} from "@t3tools/contracts"; import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; @@ -29,8 +34,11 @@ export function resolveAttachmentReferences( }); } -/** Pending uploads outlive this record only until the attachment sweep removes them. */ -const UPLOAD_OWNER_TTL_MS = 24 * 60 * 60 * 1000; +/** + * As long as the pending file can live. The sweep counts its 24 hours from when + * the upload finishes, which can be up to the upload URL's lifetime after issue. + */ +const UPLOAD_OWNER_TTL_MS = 24 * 60 * 60 * 1000 + ATTACHMENT_UPLOAD_URL_TTL_MS; /** A thread owns its uploads across provider sessions; an outside client per MCP session. */ const uploadOwner = McpInvocationContext.McpInvocationContext.pipe( diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index 5cf1953ba827..9380a8d7af90 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -14,6 +14,7 @@ import { type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as TestClock from "effect/testing/TestClock"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import { McpAttachmentInput } from "./attachment/input.ts"; @@ -727,6 +728,15 @@ it.effect("only the caller that prepared a pending upload can discard it", () => const refused = yield* call("t3_attachment_discard", { attachmentId }, otherThread); expect(declaredFailure(refused)).toMatchObject({ code: "invalid_request" }); + // Another prepare a day later keeps it: the file outlives its URL's 24 hours + // by as long as the upload took, up to the URL's own lifetime. + yield* TestClock.adjust(24 * 60 * 60 * 1000 + 5 * 60_000); + yield* call( + "t3_attachment_prepare_upload", + { upload: { name: "later.png", mimeType: "image/png", sizeBytes: 4 } }, + scope, + ); + // A new provider session of the same thread still owns the upload. const discarded = yield* call( "t3_attachment_discard",