diff --git a/apps/server/src/auth/RpcAuthorization.test.ts b/apps/server/src/auth/RpcAuthorization.test.ts index 97a8f8dc0424..0359b07b3941 100644 --- a/apps/server/src/auth/RpcAuthorization.test.ts +++ b/apps/server/src/auth/RpcAuthorization.test.ts @@ -152,6 +152,7 @@ describe("RPC authorization scopes", () => { WS_METHODS.previewReportStatus, WS_METHODS.previewAdjust, WS_METHODS.previewClearProfile, + WS_METHODS.previewReportProfiles, ]) { expect(requiredScopeForRpcMethod(method)).toBe(AuthPreviewOperateScope); } diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 5f242c439927..a33d2f3317aa 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -189,6 +189,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.previewClose]: AuthPreviewOperateScope, [WS_METHODS.previewList]: AuthOrchestrationReadScope, [WS_METHODS.previewClearProfile]: AuthPreviewOperateScope, + [WS_METHODS.previewReportProfiles]: AuthPreviewOperateScope, [WS_METHODS.previewReportStatus]: AuthPreviewOperateScope, [WS_METHODS.subscribePreviewEvents]: AuthOrchestrationReadScope, [WS_METHODS.subscribeDiscoveredLocalServers]: AuthOrchestrationReadScope, diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts index 3c2f4e5905eb..a77e8f600252 100644 --- a/apps/server/src/mcp/McpDeviceToolkit.test.ts +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { DeviceHostUnavailableError, + DeviceId, EnvironmentId, ProviderInstanceId, ThreadId, @@ -93,7 +94,21 @@ const layerDeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ platform: input.platform, openedAt: "2026-09-08T00:00:00.000Z", }), - sessionsForThread: () => Effect.succeed([]), + // UDID-1 is open in the test thread; UDID-2 exists but belongs to another thread. + sessionsForThread: (id) => + Effect.succeed( + id === threadId + ? [ + { + threadId, + hostId: "local", + deviceId: DeviceId.make("UDID-1"), + platform: "ios" as const, + openedAt: "2026-09-08T00:00:00.000Z", + }, + ] + : [], + ), screenshot: () => Effect.succeed({ device, png }), close: () => Effect.void, agentCli: Effect.succeed("/cli"), @@ -135,6 +150,12 @@ it.effect("registers the device tools and returns the screenshot as image conten screenshot: { mimeType: "image/png", width: 1206, height: 2622 }, }); + const foreign = yield* server + .callTool({ name: "device_screenshot", arguments: { deviceId: "UDID-2" } }) + .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(foreign.isError).toBe(true); + expect(foreign.content.map((entry) => entry.type)).toEqual(["text"]); + const denied = yield* server .callTool({ name: "device_list", arguments: {} }) .pipe(callWith(["preview"]), Effect.provideService(McpSchema.McpServerClient, client)); diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 3505e4e58633..fb9ca151d360 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -244,7 +244,7 @@ it.effect("tells the agent how to fall back when no desktop app can run the snap ); it.effect.each([ - { mode: "default", input: {}, images: true }, + { mode: "default", input: {}, images: false }, { mode: "explicit image", input: { includeImage: true }, images: true }, { mode: "text only", input: { includeImage: false }, images: false }, ])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) => @@ -351,12 +351,7 @@ it.effect.each([ Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provideService(McpSchema.McpServerClient, client), ); - expect(nextDefault.content.map((content) => content.type)).toEqual([ - "text", - "text", - "text", - "image", - ]); + expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "text"]); expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot }); expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree"); expect(requests).toBe(7); @@ -416,11 +411,12 @@ it.effect("saves the snapshot PNG on request and reports its path", () => const path = yield* Path.Path; const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult); - const snapshot = yield* callSnapshot({ save: true }); + const snapshot = yield* callSnapshot({ save: true, includeImage: true }); expect(snapshot.isError).toBe(false); // The browser never receives the server-only `save` flag. expect(inputs).toEqual([{}]); + expect(snapshot.content.map((content) => content.type)).toContain("image"); const structured = snapshot.structuredContent as { readonly screenshotPath?: string }; const screenshotPath = structured.screenshotPath; expect(typeof screenshotPath).toBe("string"); @@ -436,7 +432,7 @@ it.effect("saves the snapshot PNG on request and reports its path", () => expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath"); // A save without the image skips the page dump. - const pathOnly = yield* callSnapshot({ save: true, includeImage: false }); + const pathOnly = yield* callSnapshot({ save: true }); const saved = pathOnly.structuredContent as { readonly screenshotPath: string }; expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) }); expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png"); @@ -851,8 +847,8 @@ it.effect("registers annotated tools and preserves authenticated request context expect(statusTool?.tool.annotations?.destructiveHint).toBe(false); const snapshotTool = server.tools.find(({ tool }) => tool.name === "preview_snapshot"); - expect(snapshotTool?.tool.annotations?.readOnlyHint).toBe(true); - expect(snapshotTool?.tool.annotations?.idempotentHint).toBe(true); + expect(snapshotTool?.tool.annotations?.readOnlyHint).toBe(false); + expect(snapshotTool?.tool.annotations?.idempotentHint).toBe(false); expect(snapshotTool?.tool.annotations?.openWorldHint).toBe(true); const clickTool = server.tools.find(({ tool }) => tool.name === "preview_click"); @@ -891,7 +887,10 @@ it.effect("registers annotated tools and preserves authenticated request context expect(malformed._tag).toBe("InvalidParams"); const snapshot = yield* server - .callTool({ name: "preview_snapshot", arguments: { tabId: alternateTabId } }) + .callTool({ + name: "preview_snapshot", + arguments: { tabId: alternateTabId, includeImage: true }, + }) .pipe( Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provideService(McpSchema.McpServerClient, client), diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index b63cd7b95d7e..f4da6e051a7c 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -503,7 +503,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); const screenshotPath = payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; - if (screenshotPath !== undefined && payload?.includeImage === false) { + // Images stay out of tool history unless asked for: providers replay them on every + // later request, and some reject inline images outright. + const includeImage = payload?.includeImage === true; + if (screenshotPath !== undefined && !includeImage) { // The agent only wants a file to show the user. The url keeps the site icon on the tool row. const saved = { url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), @@ -548,9 +551,9 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, }, ]), - ...(payload?.includeImage === false - ? [] - : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), + ...(includeImage + ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }] + : []), ], }); }), @@ -815,7 +818,7 @@ const layerEnvironmentRegistration = toolkitRegistration( const layerProjectRegistration = toolkitRegistration(ProjectToolkit, ProjectHandlers.layer); -const layerAttachmentRegistration = toolkitRegistration( +export const layerAttachmentToolkit = toolkitRegistration( AttachmentToolkit, AttachmentHandlers.layer, ); @@ -851,7 +854,7 @@ export const layer = Layer.mergeAll( layerPreviewToolkit, layerOrchestratorToolkit, layerThreadToolkit, - layerAttachmentRegistration, + layerAttachmentToolkit, layerProjectRegistration, layerEnvironmentRegistration, layerPreviewControlsRegistration, diff --git a/apps/server/src/mcp/toolkits/attachment/handlers.ts b/apps/server/src/mcp/toolkits/attachment/handlers.ts index 9b9a6015d232..2236d8f305de 100644 --- a/apps/server/src/mcp/toolkits/attachment/handlers.ts +++ b/apps/server/src/mcp/toolkits/attachment/handlers.ts @@ -1,5 +1,12 @@ -import { type ChatAttachment, MessageId, OrchestratorMcpFailure } from "@t3tools/contracts"; +import { + ATTACHMENT_UPLOAD_URL_TTL_MS, + type ChatAttachment, + MessageId, + OrchestratorMcpFailure, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as Upload from "../../../assets/AttachmentUpload.ts"; import * as Claims from "../../../orchestration-v2/AttachmentClaims.ts"; import * as ThreadMessageIntake from "../../../orchestration-v2/ThreadMessageIntake.ts"; @@ -27,53 +34,94 @@ export function resolveAttachmentReferences( }); } -export const layer = McpToolAccess.toLayer(AttachmentToolkit, { - t3_attachment_prepare_upload: McpToolAccess.writes((input) => - Upload.issueAttachmentUploadUrl(input.upload).pipe(Effect.mapError(unavailable)), - ), - t3_attachment_discard: McpToolAccess.writes((input) => - Upload.deletePendingAttachment(input.attachmentId).pipe(Effect.as({})), - ), - t3_thread_send_attachments: McpToolAccess.writesThreads( - (input) => [input.threadId], - (input) => - Effect.gen(function* () { - const { caller, projection } = yield* readThread(input.threadId, ["messages"]); - if (projection.thread.archivedAt !== null) - return yield* new OrchestratorMcpFailure({ - code: "invalid_request", - message: "Unarchive the target thread before sending attachments.", - }); - const attachments = yield* resolveAttachmentReferences( - input.attachments, - projection.messages.flatMap((message) => message.attachments), - ); - const commandId = yield* newCommandId(); - const messageId = MessageId.make(commandId); - const result = yield* ThreadMessageIntake.sendToThread({ - projectId: projection.thread.projectId, - threadId: projection.thread.id, - commandId, - messageId, - ...(caller === undefined ? {} : { senderThreadId: caller.id }), - text: input.message ?? "", - attachments, - mode: "auto", - createdBy: "agent", - creationSource: "mcp", - }).pipe( - Effect.mapError((error) => - error._tag === "AttachmentClaimError" - ? new OrchestratorMcpFailure({ code: "orchestration_error", message: error.message }) - : unavailable(), - ), - ); - return { - threadId: projection.thread.id, - messageId, - runId: result.run.id, - status: result.run.status, - }; - }), - ), -}); +/** + * As long as the pending file can live. The sweep counts its 24 hours from when + * the upload finishes, which can be up to the upload URL's lifetime after issue. + */ +const UPLOAD_OWNER_TTL_MS = 24 * 60 * 60 * 1000 + ATTACHMENT_UPLOAD_URL_TTL_MS; + +/** A thread owns its uploads across provider sessions; an outside client per MCP session. */ +const uploadOwner = McpInvocationContext.McpInvocationContext.pipe( + Effect.map((scope) => scope.thread?.threadId ?? scope.requestNamespace), +); + +export const layer = McpToolAccess.toLayer( + AttachmentToolkit, + Effect.sync(() => { + // Which caller prepared each pending upload, so only that caller can discard it. + const uploadOwners = new Map(); + return { + t3_attachment_prepare_upload: McpToolAccess.writes((input) => + Effect.gen(function* () { + const result = yield* Upload.issueAttachmentUploadUrl(input.upload).pipe( + Effect.mapError(unavailable), + ); + const now = yield* Clock.currentTimeMillis; + for (const [id, entry] of uploadOwners) { + if (now - entry.issuedAt > UPLOAD_OWNER_TTL_MS) uploadOwners.delete(id); + } + uploadOwners.set(result.attachmentId, { owner: yield* uploadOwner, issuedAt: now }); + return result; + }), + ), + t3_attachment_discard: McpToolAccess.writes((input) => + Effect.gen(function* () { + if (uploadOwners.get(input.attachmentId)?.owner !== (yield* uploadOwner)) { + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: "Only the caller that prepared a pending upload can discard it.", + }); + } + yield* Upload.deletePendingAttachment(input.attachmentId); + uploadOwners.delete(input.attachmentId); + return {}; + }), + ), + t3_thread_send_attachments: McpToolAccess.writesThreads( + (input) => [input.threadId], + (input) => + Effect.gen(function* () { + const { caller, projection } = yield* readThread(input.threadId, ["messages"]); + if (projection.thread.archivedAt !== null) + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: "Unarchive the target thread before sending attachments.", + }); + const attachments = yield* resolveAttachmentReferences( + input.attachments, + projection.messages.flatMap((message) => message.attachments), + ); + const commandId = yield* newCommandId(); + const messageId = MessageId.make(commandId); + const result = yield* ThreadMessageIntake.sendToThread({ + projectId: projection.thread.projectId, + threadId: projection.thread.id, + commandId, + messageId, + ...(caller === undefined ? {} : { senderThreadId: caller.id }), + text: input.message ?? "", + attachments, + mode: "auto", + createdBy: "agent", + creationSource: "mcp", + }).pipe( + Effect.mapError((error) => + error._tag === "AttachmentClaimError" + ? new OrchestratorMcpFailure({ + code: "orchestration_error", + message: error.message, + }) + : unavailable(), + ), + ); + return { + threadId: projection.thread.id, + messageId, + runId: result.run.id, + status: result.run.status, + }; + }), + ), + }; + }), +); diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index 634133b62e9e..9380a8d7af90 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -14,6 +14,7 @@ import { type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as TestClock from "effect/testing/TestClock"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import { McpAttachmentInput } from "./attachment/input.ts"; @@ -26,6 +27,7 @@ import { OrchestratorProjectionError, } from "../../orchestration-v2/Orchestrator.ts"; +import * as ServerSecretStore from "../../auth/ServerSecretStore.ts"; import * as ServerConfig from "../../config.ts"; import * as ProviderAdapterRegistry from "../../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; @@ -696,3 +698,61 @@ it.effect("a caller cannot interrupt a thread that runs above its own modes", () ), ), ); + +it.effect("only the caller that prepared a pending upload can discard it", () => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const call = ( + name: string, + args: Record, + invocation: McpInvocationContext.McpInvocationScope, + ) => + server + .callTool({ name, arguments: args }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + const prepared = yield* call( + "t3_attachment_prepare_upload", + { upload: { name: "shot.png", mimeType: "image/png", sizeBytes: 4 } }, + scope, + ); + const { attachmentId } = prepared.structuredContent as { readonly attachmentId: string }; + const otherThread = { + ...scope, + requestNamespace: "other-session", + thread: { ...scope.thread!, threadId: ThreadId.make("other-thread") }, + }; + + const refused = yield* call("t3_attachment_discard", { attachmentId }, otherThread); + expect(declaredFailure(refused)).toMatchObject({ code: "invalid_request" }); + + // Another prepare a day later keeps it: the file outlives its URL's 24 hours + // by as long as the upload took, up to the URL's own lifetime. + yield* TestClock.adjust(24 * 60 * 60 * 1000 + 5 * 60_000); + yield* call( + "t3_attachment_prepare_upload", + { upload: { name: "later.png", mimeType: "image/png", sizeBytes: 4 } }, + scope, + ); + + // A new provider session of the same thread still owns the upload. + const discarded = yield* call( + "t3_attachment_discard", + { attachmentId }, + { ...scope, requestNamespace: "mcp-core-session-2" }, + ); + expect(discarded.isError).toBe(false); + }).pipe( + Effect.provide( + McpHttpServer.layerAttachmentToolkit.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provide(McpToolAccessTestkit.liveThreadsLayer), + Layer.provide(ServerSecretStore.layer), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-attachment-" })), + Layer.provide(NodeServices.layer), + ), + ), + ), +); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index bf697229718c..1543bbc87100 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -211,15 +211,20 @@ const handlers = { const scope = yield* requireDeviceAccess; const devices = yield* DeviceService.DeviceService; const sessions = yield* devices.sessionsForThread(scope.thread.threadId); - const target = - input.deviceId !== undefined - ? { hostId: input.hostId ?? LOCAL_DEVICE_HOST_ID, deviceId: input.deviceId } - : sessions - .filter((session) => input.hostId === undefined || session.hostId === input.hostId) - .at(-1); + // Only devices this thread opened: another thread's device is not this agent's to watch. + const hostId = + input.deviceId === undefined ? input.hostId : (input.hostId ?? LOCAL_DEVICE_HOST_ID); + const target = sessions.findLast( + (session) => + (hostId === undefined || session.hostId === hostId) && + (input.deviceId === undefined || session.deviceId === input.deviceId), + ); if (!target) { return yield* new DeviceToolUnavailableError({ - reason: "No device is open in this thread. Call device_open first.", + reason: + input.deviceId === undefined + ? "No device is open in this thread. Call device_open first." + : `Device ${input.deviceId} on host ${hostId} is not open in this thread. Call device_open first.`, }); } const shot = yield* devices.screenshot(target); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 9c94a1d446c6..6e34a330025e 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -64,7 +64,7 @@ const readonlyBrowserTool = (tool: T): T => const PreviewStatusTool = Tool.make("preview_status", { description: - "Report whether a collaborative browser tab is automation-capable, including its control owner, pending dialog, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.", + "Report whether a collaborative browser tab is automation-capable, including its control owner, pending dialog, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Server hosts also list every tab in the thread (tabs) with its owner, including tabs the user opened, and the browser profiles preview_open accepts. You can read any listed tab; act only on your own tabs, or on an unclaimed tab while no human controls it.", parameters: PreviewAutomationTabTargetInput, success: PreviewAutomationStatus, failure: PreviewToolFailure, @@ -78,7 +78,7 @@ const PreviewStatusTool = Tool.make("preview_status", { const PreviewOpenTool = browserTool( Tool.make("preview_open", { description: - "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Parallel subagents sharing a provider session must each open with reuseExistingTab=false and pass their returned tabId on every call. Server tabs use isolated storage and cannot be operated by a different agent session.", + "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Parallel subagents sharing a provider session must each open with reuseExistingTab=false and pass their returned tabId on every call. Pass profileId (an id or name from preview_status profiles) to open under a browser profile and its saved logins; omit it for the user's default profile. Another agent session's tabs can be read but not operated.", parameters: PreviewAutomationOpenInput, success: PreviewAutomationStatus, failure: PreviewToolFailure, @@ -91,7 +91,7 @@ const PreviewOpenTool = browserTool( const PreviewDialogTool = browserTool( Tool.make("preview_dialog", { description: - "Accept or dismiss the server browser dialog reported by preview_status. For a prompt, supply promptText when accepting. Requires this agent to own the tab. Desktop hosts may not support this operation.", + "Accept or dismiss the server browser dialog reported by preview_status. For a prompt, supply promptText when accepting. Requires this agent to own the tab, or the tab to be unclaimed with no human in control. Desktop hosts may not support this operation.", parameters: PreviewAutomationDialogInput, success: PreviewAutomationStatus, failure: PreviewToolFailure, @@ -139,22 +139,23 @@ const PreviewSetAppearanceTool = safeBrowserTool( .annotate(Tool.Idempotent, true), ); -export const PreviewSnapshotTool = readonlyBrowserTool( +// Not read-only: save=true writes the screenshot to disk. +export const PreviewSnapshotTool = safeBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab, including one the user opened (see preview_status tabs), even while they control it; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Returns page state, semantic elements, diagnostics, action history, and screenshot dimensions. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( Schema.Boolean.annotate({ description: - "Include the PNG image in the tool response. Defaults to true. Set false for text-only output.", + "Include the PNG image in the tool response. Defaults to false. Set true only when you need to see the page.", }), ), save: Schema.optional( Schema.Boolean.annotate({ description: - "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.", + "Write the screenshot PNG to disk and return its absolute path as screenshotPath. Unless includeImage=true, return only the url and screenshotPath. Defaults to false.", }), ), }), @@ -270,7 +271,7 @@ export const PreviewEvaluateResult = Schema.Struct({ const PreviewEvaluateTool = browserTool( Tool.make("preview_evaluate", { description: - "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.", + "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Needs the same control as clicking: it works on your own tabs and on the user's tab only while nobody controls it. To read a tab the user is driving, use preview_snapshot. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.", parameters: PreviewAutomationEvaluateInput, success: PreviewEvaluateResult, failure: PreviewToolFailure, diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 9aac9be1a56b..b1abaf8d465c 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -183,6 +183,7 @@ const RPC_AGGREGATES = { [WS_METHODS.previewClose]: "preview", [WS_METHODS.previewList]: "preview", [WS_METHODS.previewClearProfile]: "preview", + [WS_METHODS.previewReportProfiles]: "preview", [WS_METHODS.previewReportStatus]: "preview", [WS_METHODS.subscribePreviewEvents]: "preview", [WS_METHODS.subscribeDiscoveredLocalServers]: "preview", diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 45a2167ca042..f66e255c51b4 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -38,7 +38,8 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ constructor(options: { onContextClose?: (context: BrowserContext) => void }) { this.onClose = options.onContextClose; } - async contextFor() { + async contextFor(profileId: string, isolationKey?: string) { + contextRequests.push({ profileId, isolated: isolationKey !== undefined }); if (contextFailure) throw contextFailure; await contextGate?.promise; const context = makeContext(this.onClose); @@ -149,6 +150,8 @@ function makeContext(onClose?: (context: BrowserContext) => void) { } const contexts: ReturnType[] = []; +/** The profile and isolation each headless tab asked its context for. */ +const contextRequests: Array<{ profileId: string; isolated: boolean }> = []; let contextGate: PromiseWithResolvers | null = null; type ClipboardBinding = (source: { page: unknown }, text: unknown) => void; let clipboardBinding: ClipboardBinding | null = null; @@ -266,6 +269,7 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({ beforeEach(() => { contexts.length = 0; + contextRequests.length = 0; contextGate = null; contextFailure = null; desktopTabs.clear(); @@ -380,18 +384,37 @@ it.live("enforces provider ownership and explicit targets when a session has mul .invoke({ scope: asSession("agent-b"), tabId, - operation: "evaluate", - input: { expression: "foreign()" }, + operation: "navigate", + input: { url: "http://localhost:5173/foreign" }, }) .pipe(Effect.flip); expect(foreign).toMatchObject({ _tag: "PreviewAutomationControlInterruptedError", reason: "agentMismatch", }); - expect(contexts[0]!.sessions[0]!.send).not.toHaveBeenCalledWith( - "Runtime.evaluate", + expect(contexts[0]!.page.goto).not.toHaveBeenCalledWith( + "http://localhost:5173/foreign", expect.anything(), ); + // Another session may still read the tab, but not run page script in it. + yield* broker.invoke({ + scope: asSession("agent-b"), + tabId, + operation: "snapshot", + input: {}, + }); + const foreignEvaluate = yield* broker + .invoke({ + scope: asSession("agent-b"), + tabId, + operation: "evaluate", + input: { expression: "read()" }, + }) + .pipe(Effect.flip); + expect(foreignEvaluate).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "agentMismatch", + }); yield* broker.invoke({ scope, operation: "open", @@ -1323,6 +1346,179 @@ it.live("a desktop page the desktop takes back reconnects instead of closing", ( ).pipe(Effect.provide(layer)), ); +it.live("agents read a human's tab with no arguments and act on it only while nobody drives", () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + const opened = yield* manager.open({ + threadId: scope.thread.threadId, + url: "http://localhost:5173/mine", + runtime: "server", + }); + const tabId = PreviewTabId.make(opened.tabId); + // The user opened and is looking at the tab; attaching takes control. + const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); + const status = yield* broker.invoke({ + scope, + operation: "status", + input: {}, + }); + expect(status).toMatchObject({ + tabId, + control: { owner: "human", ownedByCaller: false }, + }); + expect(status.tabs).toEqual([ + expect.objectContaining({ tabId, owner: "human", ownedByCaller: false, visible: true }), + ]); + // Reads work while the user drives; page script does not, since it can change the page. + const evaluated = yield* broker + .invoke({ scope, operation: "evaluate", input: { expression: "read()" } }) + .pipe(Effect.flip); + expect(evaluated).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "humanControl", + }); + yield* broker.invoke({ + scope, + tabId, + operation: "snapshot", + input: {}, + }); + // Acting is refused while the user controls the tab. + const refused = yield* broker + .invoke({ + scope, + tabId, + operation: "navigate", + input: { url: "http://localhost:5173/agent" }, + }) + .pipe(Effect.flip); + expect(refused).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "humanControl", + }); + // Once they let go, the tab is unclaimed and the agent may act on it. + yield* viewer.input({ type: "releaseControl" }); + const released = yield* broker.invoke({ + scope, + tabId, + operation: "status", + input: {}, + }); + expect(released.control).toMatchObject({ owner: "unclaimed", ownedByCaller: false }); + yield* broker.invoke({ + scope, + tabId, + operation: "navigate", + input: { url: "http://localhost:5173/agent" }, + }); + expect(contexts[0]!.page.goto).toHaveBeenCalledWith( + "http://localhost:5173/agent", + expect.anything(), + ); + // Taking control back refuses the agent again. + yield* viewer.input({ type: "takeControl" }); + const retaken = yield* broker + .invoke({ scope, tabId, operation: "press", input: { key: "Enter" } }) + .pipe(Effect.flip); + expect(retaken).toMatchObject({ reason: "humanControl" }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a session's own tab stays its default while other sessions' tabs are listed", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const other = yield* broker.invoke({ + scope: asSession("agent-b"), + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + const status = yield* broker.invoke({ + scope, + operation: "status", + input: {}, + }); + expect(status.tabId).toBe(tabId); + expect(status.tabs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ tabId, owner: "agent", ownedByCaller: true }), + expect.objectContaining({ tabId: other.tabId, owner: "agent", ownedByCaller: false }), + ]), + ); + }), + ).pipe(Effect.provide(layer)), +); + +it.live( + "preview_open picks a reported profile by id or name and defaults to the reported one", + () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + yield* browser.reportProfiles({ + profiles: [{ id: "profile-work", name: "Work", kind: "persistent" }], + defaultProfileId: "profile-work", + }); + const openWith = (profileId?: string) => + broker.invoke({ + scope, + operation: "open", + input: { + reuseExistingTab: false, + show: false, + ...(profileId === undefined ? {} : { profileId }), + }, + }); + const byDefault = yield* openWith(); + const byName = yield* openWith("WORK"); + const byId = yield* openWith("incognito"); + const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); + const profileOf = (tabId: string | null) => + sessions.find((session) => session.tabId === tabId)?.profileId; + expect(profileOf(byDefault.tabId)).toBe("profile-work"); + expect(profileOf(byName.tabId)).toBe("profile-work"); + expect(profileOf(byId.tabId)).toBe("incognito"); + // The chosen profile reaches the headless tab's storage. + expect(contextRequests).toEqual([ + { profileId: "profile-work", isolated: false }, + { profileId: "profile-work", isolated: false }, + { profileId: "incognito", isolated: true }, + ]); + expect(byDefault).toMatchObject({ + defaultProfileId: "profile-work", + profiles: [ + { id: "default", name: "Default" }, + { id: "incognito", name: "Incognito", incognito: true }, + { id: "profile-work", name: "Work" }, + ], + }); + const unknown = yield* openWith("Personal").pipe(Effect.flip); + expect(unknown).toMatchObject({ + _tag: "PreviewAutomationExecutionError", + reason: + 'No browser profile is named "Personal". Use one of: Default (id default), Incognito (id incognito), Work (id profile-work).', + }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("an agent tab keeps throwaway storage when no client reported profiles", () => + Effect.scoped( + Effect.gen(function* () { + yield* ready; + expect(contextRequests).toEqual([{ profileId: "default", isolated: true }]); + }), + ).pipe(Effect.provide(layer)), +); + it.live("a desktop page that comes back reconnects without waiting for a viewer", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 7ca2e1ffba8e..f3d9bea8b40f 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -2,8 +2,14 @@ // Screencasts ignore emulated device scale; real 2x keeps captures sharp. // --disable-gpu uses cheaper software compositing while preserving SwiftShader WebGL. import { + BUILT_IN_BROWSER_PROFILES, + DEFAULT_BROWSER_PROFILE_ID, FILL_PREVIEW_VIEWPORT, + findBrowserProfile, INCOGNITO_BROWSER_PROFILE_ID, + resolveBrowserProfiles, + type BrowserProfile, + type PreviewReportProfilesInput, PREVIEW_AUTOMATION_SERVER_OPERATIONS, PreviewViewportSetting as PreviewViewportSettingSchema, PROVIDER_SEND_TURN_MAX_FILE_BYTES, @@ -243,6 +249,11 @@ export class ServerBrowser extends Context.Service< }) => Effect.Effect>; /** Deletes a human profile's server-side storage, closing its open tabs first. */ readonly clearProfile: (profileId: string) => Effect.Effect; + /** + * Records a client's browser profiles for agents' preview_open. In memory: + * clients report again on connect and whenever the list changes. + */ + readonly reportProfiles: (input: PreviewReportProfilesInput) => Effect.Effect; } >()("t3/preview/ServerBrowser") {} @@ -346,6 +357,17 @@ interface ServerDownload { } /** One agent session and the whole server; each tab holds a renderer process. */ +/** + * Operations an agent may run on any tab in its thread, even while a human + * drives it. Evaluate is not one: page script can change anything, so it needs + * the same control as clicking. + */ +const READ_OPERATIONS: ReadonlySet = new Set([ + "status", + "snapshot", + "waitFor", +]); + const AGENT_TAB_LIMIT = 8; const SERVER_TAB_LIMIT = 32; /** Agent tabs nobody watches close after this long without an agent request. */ @@ -727,10 +749,11 @@ const make = Effect.gen(function* () { adopted === undefined && (await desktopRenders(snapshot)) ? await connectDesktop(snapshot) : null; + // An agent tab without a profile (no client reported one) keeps throwaway storage. const isolatedContext = adopted === undefined && desktop === null && - (snapshot.automationOwner !== undefined || + ((snapshot.automationOwner !== undefined && snapshot.profileId === undefined) || snapshot.profileId === INCOGNITO_BROWSER_PROFILE_ID); const context = adopted?.page.context() ?? @@ -1199,10 +1222,75 @@ const make = Effect.gen(function* () { } }; - const latestThreadTab = (threadId: string, agentSessionId?: string) => - [...tabs.values()] - .filter((tab) => tab.threadId === threadId && tab.control.agentId === agentSessionId) - .sort((left, right) => right.createdAt - left.createdAt)[0]; + /** + * The tab a request without a tabId means: the caller's newest tab, else the + * thread's tab the user is looking at or used last, so "this page" works. + */ + const latestThreadTab = (threadId: string, agentSessionId?: string) => { + const threadTabs = [...tabs.values()].filter((tab) => tab.threadId === threadId); + return ( + threadTabs + .filter((tab) => tab.control.agentId === agentSessionId) + .sort((left, right) => right.createdAt - left.createdAt)[0] ?? + threadTabs.sort( + (left, right) => + Number(right.viewers.size > 0) - Number(left.viewers.size > 0) || + right.usedAt - left.usedAt, + )[0] + ); + }; + + const tabOwner = (tab: ServerTab) => + tab.control.controller !== null + ? ("human" as const) + : tab.control.agentId !== null + ? ("agent" as const) + : ("unclaimed" as const); + + /** The latest profile list a client reported; agents choose from it. */ + let reportedProfiles: { + readonly profiles: ReadonlyArray; + readonly defaultProfileId: string; + } | null = null; + + const reportProfiles: ServerBrowser["Service"]["reportProfiles"] = (input) => + Effect.sync(() => { + const profiles = resolveBrowserProfiles(input.profiles); + reportedProfiles = { + profiles, + defaultProfileId: + findBrowserProfile(profiles, input.defaultProfileId)?.id ?? DEFAULT_BROWSER_PROFILE_ID, + }; + }); + + const profileStatus = () => { + const profiles = reportedProfiles?.profiles ?? BUILT_IN_BROWSER_PROFILES; + return { + profiles: profiles.map((profile) => ({ + id: profile.id, + name: profile.name, + ...(profile.kind === "incognito" ? { incognito: true } : {}), + })), + ...(reportedProfiles ? { defaultProfileId: reportedProfiles.defaultProfileId } : {}), + }; + }; + + /** Matches an id, then a case-insensitive name. Omitted means the reported default. */ + const resolveOpenProfile = (requested: string | undefined): string | undefined => { + if (requested === undefined) return reportedProfiles?.defaultProfileId; + const profiles = reportedProfiles?.profiles ?? BUILT_IN_BROWSER_PROFILES; + const wanted = requested.toLowerCase(); + const match = + findBrowserProfile(profiles, requested) ?? + profiles.find((profile) => profile.name.toLowerCase() === wanted); + if (match) return match.id; + throw new ServerBrowserPage.ServerBrowserOperationError( + "PreviewAutomationUnknownProfileError", + `No browser profile is named "${requested}". Use one of: ${profiles + .map((profile) => `${profile.name} (id ${profile.id})`) + .join(", ")}.`, + ); + }; const statusWithTitle = async ( tab: ServerTab | undefined, @@ -1216,6 +1304,7 @@ const make = Effect.gen(function* () { url: null, title: null, loading: false, + ...profileStatus(), }; } const url = tab.page.url(); @@ -1228,12 +1317,7 @@ const make = Effect.gen(function* () { title: null, loading: tab.loading, control: { - owner: - tab.control.controller !== null - ? ("human" as const) - : tab.control.agentId !== null - ? ("agent" as const) - : ("unclaimed" as const), + owner: tabOwner(tab), ownedByCaller: tab.control.agentId === agentSessionId, generation: tab.control.generation, }, @@ -1244,15 +1328,17 @@ const make = Effect.gen(function* () { viewportSetting: tab.setting, ...(viewport ? { viewport } : {}), tabs: [...tabs.values()] - .filter( - (candidate) => - candidate.threadId === tab.threadId && candidate.control.agentId === agentSessionId, - ) + .filter((candidate) => candidate.threadId === tab.threadId) .map((candidate) => ({ tabId: candidate.tabId, url: candidate.page.url() === "about:blank" ? null : candidate.page.url(), ...(candidate.openerTabId === undefined ? {} : { openerTabId: candidate.openerTabId }), + owner: tabOwner(candidate), + ownedByCaller: candidate.control.agentId === agentSessionId, + visible: candidate.viewers.size > 0, + ...(candidate.profileId === undefined ? {} : { profileId: candidate.profileId }), })), + ...profileStatus(), downloads: tab.downloads.map(({ fileName, path, sizeBytes, url, completedAt }) => ({ fileName, path, @@ -1554,19 +1640,26 @@ const make = Effect.gen(function* () { "No server preview tab is open for this thread. Call preview_open first.", ); } - if (tab.control.agentId !== request.agentSessionId) + // Any tab in the thread can be read; acting is checked by its control. + if (!READ_OPERATIONS.has(request.operation) && !tab.control.agentMayAct(request.agentSessionId)) throw new BrowserControlInterrupted( - "This tab belongs to another agent session or a human. Open your own tab.", + "This tab belongs to another agent session. You can read it, but open your own tab to act.", "agentMismatch", ); return tab; }; - /** Any request, even a failed one, keeps the agent's tabs on the thread from idling out. */ + /** + * Any request, even a failed one, keeps the agent's tabs on the thread from + * idling out. The tab it targeted becomes the thread's most recently used. + */ const markUsed = (request: PreviewAutomationRequest) => { const now = Date.now(); for (const tab of tabs.values()) { - if (tab.threadId === request.threadId && tab.control.agentId === request.agentSessionId) + if ( + tab.threadId === request.threadId && + (tab.control.agentId === request.agentSessionId || tab.tabId === request.tabId) + ) tab.usedAt = now; } }; @@ -1602,7 +1695,7 @@ const make = Effect.gen(function* () { "tabRequired", ); // A tab still launching exists only as a session, so resolve it like a viewer would. - const existing = + const found = reuse && request.tabId !== undefined ? await Effect.runPromise( findTab(request.threadId, request.tabId).pipe( @@ -1612,7 +1705,14 @@ const make = Effect.gen(function* () { ), ) : undefined; + // Only an explicit tabId reuses a tab this session did not open; a tab + // it last read (such as the user's) is not taken over implicitly. + const existing = + found && (request.tabIdExplicit || found.control.agentId === request.agentSessionId) + ? found + : undefined; const navigationTimeout = Math.min(request.timeoutMs, NAVIGATION_TIMEOUT_MS); + const profileId = existing ? undefined : resolveOpenProfile(open.profileId); if (!existing) { closeIdleAgentTabs(); assertTabCapacity(request.agentSessionId); @@ -1624,6 +1724,7 @@ const make = Effect.gen(function* () { manager.open({ threadId: request.threadId, ...(url ? { url } : {}), + ...(profileId === undefined ? {} : { profileId }), runtime: "server", reveal: false, automationOwner: request.agentSessionId, @@ -1681,7 +1782,7 @@ const make = Effect.gen(function* () { const recordings = [...tabs.values()].filter( (candidate) => candidate.threadId === request.threadId && - candidate.control.agentId === request.agentSessionId && + candidate.control.agentMayAct(request.agentSessionId ?? "") && (candidate.recording || candidate.recordingStart), ); const targetTabId = @@ -1701,6 +1802,11 @@ const make = Effect.gen(function* () { const tab = await requireTab(request); // Closing must unblock an action waiting on a dialog, without queueing behind it. if (request.operation === "close") { + if (tab.control.agentId !== request.agentSessionId) + throw new BrowserControlInterrupted( + "Only the agent session that opened this tab can close it.", + "agentMismatch", + ); if (tab.control.controller !== null) throw new BrowserControlInterrupted("A human controls this tab.", "humanControl"); void tab.control.close().catch(constVoid); @@ -1716,7 +1822,26 @@ const make = Effect.gen(function* () { await resolveDialog(tab, input as PreviewAutomationDialogInput); return statusWithTitle(tab, request.agentSessionId); } - return tab.control.agent(request.agentSessionId!, async () => { + const agentSessionId = request.agentSessionId!; + // A tab this session did not open, while it cannot act there (another + // agent's, or the user's while they drive): read it in place, without + // queueing behind the human's input. Its own tabs keep the takeover + // contract: a human taking control interrupts every agent call. + if ( + READ_OPERATIONS.has(request.operation) && + tab.control.agentId !== agentSessionId && + !tab.control.agentCanActNow(agentSessionId) + ) { + return tab.control.observe(async () => { + if (tab.dialog) + throw new BrowserControlInterrupted( + "A browser dialog is pending. Read preview_status.", + "dialogPending", + ); + return executeTabOperation(tab, request); + }); + } + return tab.control.agent(agentSessionId, async () => { if (tab.dialog) throw new BrowserControlInterrupted( "A browser dialog is pending. Read preview_status and use preview_dialog first.", @@ -2328,6 +2453,7 @@ const make = Effect.gen(function* () { return ServerBrowser.of({ attachViewer, clearProfile, + reportProfiles, openDownload, answerFileChooser, }); diff --git a/apps/server/src/preview/ServerBrowserStream.test.ts b/apps/server/src/preview/ServerBrowserStream.test.ts index e3366178b668..0654ea62a2b8 100644 --- a/apps/server/src/preview/ServerBrowserStream.test.ts +++ b/apps/server/src/preview/ServerBrowserStream.test.ts @@ -87,6 +87,7 @@ it.effect.each([ }); const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: () => Effect.succeedNone, answerFileChooser: () => Effect.succeed(false), attachViewer: (input) => @@ -163,6 +164,7 @@ it.effect.each([ let attachments = 0; const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: () => Effect.succeedNone, answerFileChooser: () => Effect.succeed(false), attachViewer: () => { @@ -203,6 +205,7 @@ it.effect("serves a tab's download only to an authorized session", () => const requests: Array = []; const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: (input) => Effect.sync(() => { requests.push(input); @@ -250,6 +253,7 @@ it.effect("passes uploaded files to the page's open picker and needs operate sco const answers: Array<{ chooserId: string; files: Array<{ name: string; text: string }> }> = []; const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: () => Effect.succeedNone, answerFileChooser: (input) => Effect.sync(() => { @@ -317,6 +321,7 @@ it.effect.each([ Effect.gen(function* () { const browser = ServerBrowser.ServerBrowser.of({ clearProfile: () => Effect.void, + reportProfiles: () => Effect.void, openDownload: () => Effect.succeedNone, answerFileChooser: () => Effect.succeed(false), attachViewer: () => Effect.fail(new ServerBrowser.ServerBrowserLaunchError({ cause: error })), diff --git a/apps/server/src/preview/SessionControl.test.ts b/apps/server/src/preview/SessionControl.test.ts index 73d64cee1019..42eabc212ac0 100644 --- a/apps/server/src/preview/SessionControl.test.ts +++ b/apps/server/src/preview/SessionControl.test.ts @@ -166,4 +166,16 @@ describe("SessionControl", () => { ).rejects.toThrow("navigation failed"); await expect(control.agent("agent", async () => "recovered")).resolves.toBe("recovered"); }); + + it("lets any agent act on a tab no agent opened, but only while no human controls it", async () => { + const control = new SessionControl(null); + await expect(control.agent("agent-a", async () => "acted")).resolves.toBe("acted"); + await control.take("viewer-a"); + await expect(control.agent("agent-a", async () => "racing")).rejects.toMatchObject({ + reason: "humanControl", + }); + await expect(control.observe(async () => "read")).resolves.toBe("read"); + await control.release("viewer-a"); + await expect(control.agent("agent-b", async () => "acted")).resolves.toBe("acted"); + }); }); diff --git a/apps/server/src/preview/SessionControl.ts b/apps/server/src/preview/SessionControl.ts index a361cd1a467c..a930a5e115fc 100644 --- a/apps/server/src/preview/SessionControl.ts +++ b/apps/server/src/preview/SessionControl.ts @@ -73,8 +73,17 @@ export class SessionControl { }); } + /** Whether an agent may act here: its own tab, or a tab no agent opened. */ + agentMayAct(agentId: string) { + return this.agentId === null || this.agentId === agentId; + } + + /** + * An agent action. It may act on its own tab or on one a human opened, but + * never while a human controls the tab; taking control interrupts it. + */ agent(agentId: string, run: () => Promise) { - if (this.agentId !== agentId) + if (!this.agentMayAct(agentId)) return Promise.reject( new BrowserControlInterrupted("This tab belongs to another agent.", "agentMismatch"), ); @@ -82,7 +91,21 @@ export class SessionControl { return Promise.reject( new BrowserControlInterrupted("A human controls this tab.", "humanControl"), ); - return this.action(() => this.agentId === agentId && this.owner === null, run); + return this.action(() => this.agentMayAct(agentId) && this.owner === null, run); + } + + /** Whether this agent's action would run now rather than be refused. */ + agentCanActNow(agentId: string) { + return this.agentMayAct(agentId) && this.owner === null; + } + + /** + * A read that needs no control and does not queue, so an agent can look at + * a page while a human drives it without holding up the human's input. + */ + observe(run: () => Promise) { + this.assertOpen(); + return run(); } /** diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 5e2dfc6b5dc7..eee2755fcf08 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2899,6 +2899,7 @@ const layerWsRpc = ( [WS_METHODS.previewClose]: (input) => previewManager.close(input), [WS_METHODS.previewList]: (input) => previewManager.list(input), [WS_METHODS.previewClearProfile]: (input) => serverBrowser.clearProfile(input.profileId), + [WS_METHODS.previewReportProfiles]: (input) => serverBrowser.reportProfiles(input), [WS_METHODS.previewReportStatus]: (input) => previewManager.reportStatus(input), [WS_METHODS.subscribePreviewEvents]: (_input) => previewManager.events, [WS_METHODS.deviceConfigure]: (input) => deviceService.configure(input), diff --git a/apps/web/src/AppRoot.tsx b/apps/web/src/AppRoot.tsx index 443c260f5e86..1ec12cfdd712 100644 --- a/apps/web/src/AppRoot.tsx +++ b/apps/web/src/AppRoot.tsx @@ -1,5 +1,6 @@ import { RouterProvider } from "@tanstack/react-router"; +import { BrowserProfileReporter } from "./browser/BrowserProfileReporter"; import { ElectronBrowserHost } from "./browser/ElectronBrowserHost"; import { QuitHoldOverlay } from "./components/QuitHoldOverlay"; import { AppAtomRegistryProvider } from "./rpc/atomRegistry"; @@ -15,6 +16,7 @@ export function AppRoot({ router }: { readonly router: AppRouter }) { + ); diff --git a/apps/web/src/browser/BrowserProfileReporter.tsx b/apps/web/src/browser/BrowserProfileReporter.tsx new file mode 100644 index 000000000000..e148f79d72af --- /dev/null +++ b/apps/web/src/browser/BrowserProfileReporter.tsx @@ -0,0 +1,40 @@ +import { AuthPreviewOperateScope, type BrowserProfile } from "@t3tools/contracts"; +import { useEffect } from "react"; + +import { useClientSettings, useClientSettingsHydrated } from "~/hooks/useSettings"; +import { useServerConfigs } from "~/state/entities"; +import { useEnvironments } from "~/state/environments"; +import { previewEnvironment } from "~/state/preview"; +import { useEnvironmentsWithScope } from "~/state/session"; +import { useAtomCommand } from "~/state/use-atom-command"; + +const selectProfiles = (settings: { readonly browserProfiles: ReadonlyArray }) => + settings.browserProfiles; +const selectDefaultProfileId = (settings: { readonly browserDefaultProfileId: string }) => + settings.browserDefaultProfileId; + +/** + * Tells each environment that hosts browser tabs which profiles this client + * has, so agents can open tabs under them. Profiles live in client settings; + * the server keeps only the latest report in memory. A reconnect brings a new + * server config, which sends the report again. + */ +export function BrowserProfileReporter() { + const hydrated = useClientSettingsHydrated(); + const profiles = useClientSettings(selectProfiles); + const defaultProfileId = useClientSettings(selectDefaultProfileId); + const { environments } = useEnvironments(); + const serverConfigs = useServerConfigs(); + const operable = useEnvironmentsWithScope(environments, AuthPreviewOperateScope); + const report = useAtomCommand(previewEnvironment.reportProfiles, { reportFailure: false }); + + useEffect(() => { + if (!hydrated) return; + for (const [environmentId, config] of serverConfigs) { + if (!config.environment.capabilities.serverBrowser || !operable.has(environmentId)) continue; + void report({ environmentId, input: { profiles, defaultProfileId } }); + } + }, [defaultProfileId, hydrated, operable, profiles, report, serverConfigs]); + + return null; +} diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index e32dcb9ee4df..b076a30a5221 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -3,11 +3,12 @@ import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; import { AuthPreviewOperateScope, FILL_PREVIEW_VIEWPORT } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; -import { type ComponentProps, useEffect, useMemo, useRef } from "react"; +import { type ComponentProps, useEffect, useMemo, useRef, useState } from "react"; import { primaryEnvironmentIdAtom } from "~/state/primaryEnvironment"; import { isElectron } from "~/env"; +import { useClientSettingsHydrated } from "~/hooks/useSettings"; import { useTheme } from "~/hooks/useTheme"; import { useActivePreviewSessions } from "~/previewStateStore"; import { previewEnvironment } from "~/state/preview"; @@ -15,6 +16,7 @@ import { useEnvironmentScope } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; import { readPreviewAnnotationTheme } from "./annotationTheme"; +import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; import { HostedBrowserWebview } from "./HostedBrowserWebview"; import { openUrlInPreview } from "./openFileInPreview"; @@ -158,5 +160,23 @@ function AuthorizedBrowserWebview(props: ComponentProps : null; + const profileId = useTabProfileId(props.profileId); + return canOperatePreview ? : null; +} + +/** + * Agent `preview_open` normally carries the profile clients reported (see + * BrowserProfileReporter). An agent tab opened before any client reported has + * none, so it falls back to the configured default here. It is latched once + * settings load: Electron fixes the partition when the guest attaches, so a + * later settings change must not move a live tab. + */ +function useTabProfileId(profileId: string | undefined): string | undefined { + const hydrated = useClientSettingsHydrated(); + const defaultProfileId = useBrowserDefaults().profileId; + const [fallback, setFallback] = useState(undefined); + if (profileId === undefined && hydrated && fallback === undefined) { + setFallback(defaultProfileId); + } + return profileId ?? fallback; } diff --git a/packages/client-runtime/src/state/preview.ts b/packages/client-runtime/src/state/preview.ts index 69ed75b9ea65..72d47b742699 100644 --- a/packages/client-runtime/src/state/preview.ts +++ b/packages/client-runtime/src/state/preview.ts @@ -76,6 +76,11 @@ export function createPreviewEnvironmentAtoms( label: "environment-data:preview:clear-profile", tag: WS_METHODS.previewClearProfile, }), + reportProfiles: createEnvironmentRpcCommand(runtime, { + label: "environment-data:preview:report-profiles", + tag: WS_METHODS.previewReportProfiles, + concurrency: { mode: "latest", key: ({ environmentId }) => environmentId }, + }), reportStatus: createEnvironmentRpcCommand(runtime, { label: "environment-data:preview:report-status", tag: WS_METHODS.previewReportStatus, diff --git a/packages/contracts/src/preview.ts b/packages/contracts/src/preview.ts index e534fff06984..802bbd4765b7 100644 --- a/packages/contracts/src/preview.ts +++ b/packages/contracts/src/preview.ts @@ -10,7 +10,7 @@ */ import { Schema } from "effect"; import { NonNegativeInt, PositiveInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; -import { BrowserProfileId } from "./browserProfile.ts"; +import { BROWSER_PROFILE_MAX_COUNT, BrowserProfile, BrowserProfileId } from "./browserProfile.ts"; export const PREVIEW_URL_MAX_LENGTH = 2_048; export const CONFIGURED_LOCAL_SERVER_URLS_MAX_ITEMS = 32; @@ -295,6 +295,14 @@ export const PreviewCloseInput = Schema.Struct({ }); export type PreviewCloseInput = typeof PreviewCloseInput.Type; +/** A client's browser profiles, which agents choose from when they open a tab. */ +export const PreviewReportProfilesInput = Schema.Struct({ + /** The user's own profiles; the server adds the built-ins. */ + profiles: Schema.Array(BrowserProfile).check(Schema.isMaxLength(BROWSER_PROFILE_MAX_COUNT)), + defaultProfileId: BrowserProfileId, +}); +export type PreviewReportProfilesInput = typeof PreviewReportProfilesInput.Type; + export const PreviewClearProfileInput = Schema.Struct({ profileId: BrowserProfileId, }); diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index fa16511bd3e8..6634e73b6990 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -10,12 +10,15 @@ import { PreviewViewportSize, } from "./preview.ts"; import { ProviderInstanceId } from "./providerInstance.ts"; +import { BrowserProfileId } from "./browserProfile.ts"; const BoundedUrl = Schema.String.check(Schema.isTrimmed()) .check(Schema.isNonEmpty()) .check(Schema.isMaxLength(2048)); const URL_GUIDANCE = "Absolute http(s) URL or a schemeless host such as t3.chat or localhost:5173. Schemeless public hosts use https; loopback hosts use http."; +const PROFILE_GUIDANCE = + "Browser profile for a new tab, by id or name (case-insensitive) from preview_status profiles. Omit to use the user's default profile. A tab keeps the profile it opened with."; const OptionalTimeoutMs = Schema.optional( Schema.Int.check(Schema.isGreaterThan(0)) .check(Schema.isLessThanOrEqualTo(60_000)) @@ -61,17 +64,27 @@ const PreviewAutomationTabTargetFields = { tabId: Schema.optional( PreviewTabId.annotate({ description: - "Exact collaborative browser tab to target. Omit to use this agent session's current tab.", + "Exact collaborative browser tab to target, including a tab the user opened (see preview_status tabs). Omit to use this agent session's current tab, or the user's visible tab when this session has none.", }), ).annotate({ description: - "Exact collaborative browser tab to target. Omit to use this agent session's current tab.", + "Exact collaborative browser tab to target, including a tab the user opened (see preview_status tabs). Omit to use this agent session's current tab, or the user's visible tab when this session has none.", }), }; export const PreviewAutomationTabTargetInput = Schema.Struct(PreviewAutomationTabTargetFields); export type PreviewAutomationTabTargetInput = typeof PreviewAutomationTabTargetInput.Type; +/** `human`: a person controls the tab now; `agent`: an agent session opened it; `unclaimed`: neither. */ +export const PreviewAutomationTabOwner = Schema.Literals(["agent", "human", "unclaimed"]); + +export const PreviewAutomationProfile = Schema.Struct({ + id: BrowserProfileId, + name: Schema.String, + incognito: Schema.optional(Schema.Boolean), +}); +export type PreviewAutomationProfile = typeof PreviewAutomationProfile.Type; + export const PreviewAutomationStatus = Schema.Struct({ available: Schema.Boolean, visible: Schema.Boolean, @@ -81,7 +94,7 @@ export const PreviewAutomationStatus = Schema.Struct({ loading: Schema.Boolean, control: Schema.optional( Schema.Struct({ - owner: Schema.Literals(["agent", "human", "unclaimed"]), + owner: PreviewAutomationTabOwner, ownedByCaller: Schema.Boolean, generation: Schema.Number, }), @@ -115,16 +128,28 @@ export const PreviewAutomationStatus = Schema.Struct({ }), ), ), - /** Server hosts: every tab this agent session owns, including popups its pages opened. */ + /** + * Server hosts: every tab in the thread, including ones the user or another + * agent session opened. Any of them can be read; acting needs `ownedByCaller`, + * or an `unclaimed` tab while no human controls it. + */ tabs: Schema.optional( Schema.Array( Schema.Struct({ tabId: PreviewTabId, url: Schema.NullOr(Schema.String), openerTabId: Schema.optional(PreviewTabId), + owner: Schema.optional(PreviewAutomationTabOwner), + ownedByCaller: Schema.optional(Schema.Boolean), + visible: Schema.optional(Schema.Boolean), + profileId: Schema.optional(BrowserProfileId), }), ), ), + /** Server hosts: the browser profiles preview_open accepts, as the user's client reported them. */ + profiles: Schema.optional(Schema.Array(PreviewAutomationProfile)), + /** Server hosts: the profile preview_open uses when given none. */ + defaultProfileId: Schema.optional(BrowserProfileId), }); export type PreviewAutomationStatus = typeof PreviewAutomationStatus.Type; @@ -162,6 +187,11 @@ export const PreviewAutomationOpenInput = Schema.Struct({ "Reuse tabId when supplied, otherwise this agent session's current tab. Defaults to true; set false to create a new tab.", }), ), + profileId: Schema.optional( + TrimmedNonEmptyString.check(Schema.isMaxLength(64)).annotate({ + description: PROFILE_GUIDANCE, + }), + ).annotate({ description: PROFILE_GUIDANCE }), }) .check( Schema.makeFilter( diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 875223bfd1c0..ed14e2e16b7c 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -256,6 +256,7 @@ import { PreviewListResult, PreviewClearProfileError, PreviewClearProfileInput, + PreviewReportProfilesInput, PreviewNavigateInput, PreviewOpenInput, PreviewRefreshInput, @@ -444,6 +445,7 @@ export const WS_METHODS = { previewClose: "preview.close", previewList: "preview.list", previewClearProfile: "preview.clearProfile", + previewReportProfiles: "preview.reportProfiles", previewReportStatus: "preview.reportStatus", // Device methods @@ -1475,6 +1477,11 @@ const WsPreviewClearProfileRpc = Rpc.make(WS_METHODS.previewClearProfile, { error: Schema.Union([PreviewClearProfileError, EnvironmentAuthorizationError]), }); +const WsPreviewReportProfilesRpc = Rpc.make(WS_METHODS.previewReportProfiles, { + payload: PreviewReportProfilesInput, + error: EnvironmentAuthorizationError, +}); + const WsPreviewReportStatusRpc = Rpc.make(WS_METHODS.previewReportStatus, { payload: PreviewReportStatusInput, error: Schema.Union([PreviewError, EnvironmentAuthorizationError]), @@ -1955,6 +1962,7 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewCloseRpc, WsPreviewListRpc, WsPreviewClearProfileRpc, + WsPreviewReportProfilesRpc, WsPreviewReportStatusRpc, WsSubscribePreviewEventsRpc, WsSubscribeDiscoveredLocalServersRpc, diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 2280d4d25619..6dcf3875ff25 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -44,7 +44,12 @@ You are running inside T3 Code. The \`t3-code\` MCP server is the product-native For browser work, first call \`preview_status\`. If no automation-capable preview is attached, call \`preview_open\` before concluding that the browser is unavailable. Then use \`preview_navigate\`, \`preview_snapshot\`, and the focused interaction tools. Prefer snapshot-provided locators over coordinates. -Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Use an alternative browser system only when the T3 preview tools are absent, the user explicitly requests another browser, or \`preview_open\` returns an explicit unsupported/unavailable error. A failed T3 preview tool call should be inspected and retried with corrected arguments when the error is actionable. +\`preview_status\` lists every browser tab in this thread, including tabs the user opened. When the user asks about "this page" or a page they have open, read their tab: pass its \`tabId\` to \`preview_snapshot\` or \`preview_wait_for\`, or omit \`tabId\` when you have no tab of your own. You may act on the user's tab, including \`preview_evaluate\`, only while its owner is \`unclaimed\`; while it is \`human\`, the user is driving, so read it with \`preview_snapshot\` or open your own tab. To use a browser profile (a set of saved logins), pass \`profileId\` from \`preview_status\` profiles to \`preview_open\`. + +Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Inspect a failed preview call and retry with corrected arguments when the error is actionable. Use another browser system when: +- the T3 preview tools are absent, or \`preview_open\` returns an explicit unsupported/unavailable error; +- the user asks for another browser, or invokes a skill or documented repository workflow that names one; follow that workflow and report any prerequisite it is missing; +- preview calls on an open tab have failed twice on the same step (timeouts, \`chrome-error://\` pages, a different client answering). Quote the raw error and switch without asking the user which browser to use. `; const T3_CODE_ACP_DEFAULT_MODE_INSTRUCTIONS = `## T3 Code interaction mode: Default