feat(preview): give agents hover, select, drag, and click options

Agents could only left-click once, so hover menus, native dropdowns,
context menus, double-click editing, and drag-and-drop needed evaluate
hacks or were out of reach. preview_click takes button and clickCount on
server and desktop tabs, and server tabs add preview_hover,
preview_select (by value or visible label), and preview_drag.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Author: juliusmarminge

apps/desktop/src/preview/Manager.test.ts

@@ -4166,7 +4166,15 @@ describe("PreviewManager", () => {
             }
             if (method === "Input.dispatchMouseEvent" && params?.type === "mousePressed") {
               activity.push("mousePressed");
-              humanInput?.({}, { kind: "pointer", x: params.x, y: params.y, button: 0 });
+              humanInput?.(
+                {},
+                {
+                  kind: "pointer",
+                  x: params.x,
+                  y: params.y,
+                  button: { left: 0, middle: 1, right: 2 }[params.button as "left"],
+                },
+              );
             }
             return undefined;
           });
@@ -4211,6 +4219,12 @@ describe("PreviewManager", () => {
               activity.push(event.phase);
             }),
           );
+          const controllers: string[] = [];
+          yield* manager.subscribeStateChanges((_tabId, state) =>
+            Effect.sync(() => {
+              controllers.push(state.controller);
+            }),
+          );
           yield* manager.createTab("tab_1");
           yield* manager.registerWebview("tab_1", 42);
           yield* manager.startRecording("tab_1");
@@ -4249,6 +4263,25 @@ describe("PreviewManager", () => {
             button: "left",
             clickCount: 1,
           });
+
+          // A right double-click is the agent's own input, so it keeps control of the tab.
+          sendCommand.mockClear();
+          const doubleClick = yield* manager
+            .automationClick("tab_1", { x: 120, y: 80, button: "right", clickCount: 2 })
+            .pipe(Effect.forkChild({ startImmediately: true }));
+          yield* TestClock.adjust(200);
+          yield* Fiber.join(doubleClick);
+          expect(
+            sendCommand.mock.calls
+              .filter(([method]) => method === "Input.dispatchMouseEvent")
+              .map(([, params]) => [params?.type, params?.button, params?.clickCount]),
+          ).toEqual([
+            ["mousePressed", "right", 1],
+            ["mouseReleased", "right", 1],
+            ["mousePressed", "right", 2],
+            ["mouseReleased", "right", 2],
+          ]);
+          expect(controllers).not.toContain("human");
         }),
       ),
   );

apps/desktop/src/preview/Manager.ts

@@ -616,6 +616,9 @@ const isPreviewInputSignal = (value: unknown): value is PreviewInputSignal => {
   );
 };
 
+/** DOM `PointerEvent.button` for each CDP mouse button, matched against reported human input. */
+const POINTER_BUTTON_INDEX = { left: 0, middle: 1, right: 2 } as const;
+
 const inputSignalsMatch = (left: PreviewInputSignal, right: PreviewInputSignal): boolean => {
   if (left.kind !== right.kind) return false;
   if (left.kind === "pointer" && right.kind === "pointer") {
@@ -3970,19 +3973,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
       createdAt: clickCreatedAt,
     });
     yield* Effect.sleep(AGENT_CURSOR_CLICK_LEAD_MS);
-    yield* expectAgentInput(tabId, { kind: "pointer", ...point, button: 0 });
-    yield* send("Input.dispatchMouseEvent", {
-      type: "mousePressed",
-      ...point,
-      button: "left",
-      clickCount: 1,
-    });
-    yield* send("Input.dispatchMouseEvent", {
-      type: "mouseReleased",
-      ...point,
-      button: "left",
-      clickCount: 1,
-    });
+    const button = input.button ?? "left";
+    // A double-click is two press/release pairs whose clickCount climbs, as Chromium sends them.
+    for (let clickCount = 1; clickCount <= (input.clickCount ?? 1); clickCount += 1) {
+      yield* expectAgentInput(tabId, {
+        kind: "pointer",
+        ...point,
+        button: POINTER_BUTTON_INDEX[button],
+      });
+      yield* send("Input.dispatchMouseEvent", {
+        type: "mousePressed",
+        ...point,
+        button,
+        clickCount,
+      });
+      yield* send("Input.dispatchMouseEvent", {
+        type: "mouseReleased",
+        ...point,
+        button,
+        clickCount,
+      });
+    }
   });
 
   const automationClick = Effect.fn("PreviewManager.automationClick")(function* (

apps/server/src/mcp/toolkits/preview/handlers.ts

@@ -13,6 +13,7 @@ import {
   type PreviewAutomationOpenInput,
   type PreviewAutomationRecordingStatus,
   type PreviewAutomationResizeResult,
+  type PreviewAutomationSelectResult,
   type PreviewAutomationSetColorSchemeResult,
   type PreviewAutomationSnapshot,
   type PreviewAutomationStatus,
@@ -205,6 +206,10 @@ const handlers = {
   },
   preview_click: (input) => invokeTargeted<object>("click", input, input.timeoutMs),
   preview_type: (input) => invokeTargeted<object>("type", input, input.timeoutMs),
+  preview_hover: (input) => invokeTargeted<object>("hover", input, input.timeoutMs),
+  preview_select: (input) =>
+    invokeTargeted<PreviewAutomationSelectResult>("select", input, input.timeoutMs),
+  preview_drag: (input) => invokeTargeted<object>("drag", input, input.timeoutMs),
   preview_upload: (input) => invokeTargeted<object>("upload", input, input.timeoutMs),
   preview_press: (input) => invokeTargeted<object>("press", input),
   preview_scroll: (input) => invokeTargeted<object>("scroll", input),

apps/server/src/mcp/toolkits/preview/tools.test.ts

@@ -60,6 +60,8 @@ it("exports exact object result schemas for preview actions", () => {
   const actionNames = [
     "preview_click",
     "preview_type",
+    "preview_hover",
+    "preview_drag",
     "preview_upload",
     "preview_press",
     "preview_scroll",

apps/server/src/mcp/toolkits/preview/tools.ts

@@ -2,8 +2,10 @@ import {
   ToolActivityIcon,
   PreviewAutomationClickInput,
   PreviewAutomationDialogInput,
+  PreviewAutomationDragInput,
   PreviewAutomationError,
   PreviewAutomationEvaluateInput,
+  PreviewAutomationHoverInput,
   PreviewAutomationNavigateInput,
   PreviewAutomationOpenInput,
   PreviewAutomationPressInput,
@@ -12,6 +14,8 @@ import {
   PreviewAutomationResizeInput,
   PreviewAutomationResizeResult,
   PreviewAutomationScrollInput,
+  PreviewAutomationSelectInput,
+  PreviewAutomationSelectResult,
   PreviewAutomationSetColorSchemeInput,
   PreviewAutomationSetColorSchemeResult,
   PreviewAutomationSnapshot,
@@ -157,7 +161,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool(
 const PreviewClickTool = browserTool(
   Tool.make("preview_click", {
     description:
-      "Click exactly one target in the tab selected by tabId, or this agent session's current tab when omitted. Prefer a Playwright locator; selector accepts legacy CSS; x and y must be supplied together.",
+      "Click exactly one target in the tab selected by tabId, or this agent session's current tab when omitted. Prefer a Playwright locator; selector accepts legacy CSS; x and y must be supplied together. Set button=right for a context menu or clickCount=2 for a double-click; server browser tabs only.",
     parameters: PreviewAutomationClickInput,
     success: PreviewActionResult,
     failure: PreviewAutomationError,
@@ -176,6 +180,42 @@ const PreviewTypeTool = browserTool(
   }).annotate(Tool.Title, "Type into preview page"),
 );
 
+const PreviewHoverTool = safeBrowserTool(
+  Tool.make("preview_hover", {
+    description:
+      "Move the mouse over exactly one target in the tab selected by tabId, or this agent session's current tab when omitted, to reveal hover menus and tooltips. Server browser tabs only.",
+    parameters: PreviewAutomationHoverInput,
+    success: PreviewActionResult,
+    failure: PreviewAutomationError,
+    dependencies,
+  }).annotate(Tool.Title, "Hover preview page"),
+);
+
+const PreviewSelectTool = browserTool(
+  Tool.make("preview_select", {
+    description:
+      "Choose options in one native <select> in the tab selected by tabId, or this agent session's current tab when omitted, by option value or visible label. Custom dropdowns are not <select>; click them open and click the option instead. Server browser tabs only.",
+    parameters: PreviewAutomationSelectInput,
+    success: Schema.Struct({
+      ...PreviewAutomationSelectResult.fields,
+      ...presentationFields,
+    }),
+    failure: PreviewAutomationError,
+    dependencies,
+  }).annotate(Tool.Title, "Select preview option"),
+);
+
+const PreviewDragTool = browserTool(
+  Tool.make("preview_drag", {
+    description:
+      "Drag one element onto another in the tab selected by tabId, or this agent session's current tab when omitted. Server browser tabs only.",
+    parameters: PreviewAutomationDragInput,
+    success: PreviewActionResult,
+    failure: PreviewAutomationError,
+    dependencies,
+  }).annotate(Tool.Title, "Drag in preview page"),
+);
+
 const PreviewUploadTool = browserTool(
   Tool.make("preview_upload", {
     description:
@@ -275,6 +315,9 @@ export const PreviewToolkit = Toolkit.make(
   PreviewSnapshotTool,
   PreviewClickTool,
   PreviewTypeTool,
+  PreviewHoverTool,
+  PreviewSelectTool,
+  PreviewDragTool,
   PreviewUploadTool,
   PreviewPressTool,
   PreviewScrollTool,
@@ -293,6 +336,9 @@ export const PreviewStandardToolkit = Toolkit.make(
   PreviewSetAppearanceTool,
   PreviewClickTool,
   PreviewTypeTool,
+  PreviewHoverTool,
+  PreviewSelectTool,
+  PreviewDragTool,
   PreviewUploadTool,
   PreviewPressTool,
   PreviewScrollTool,

apps/server/src/preview/ServerBrowser.ts

@@ -11,14 +11,17 @@ import {
   type PreviewAutomationClickInput,
   type PreviewAutomationDialogInput,
   type PreviewAutomationConsoleEntry,
+  type PreviewAutomationDragInput,
   type PreviewAutomationEvaluateInput,
+  type PreviewAutomationHoverInput,
   type PreviewAutomationNavigateInput,
   type PreviewAutomationNetworkEntry,
   type PreviewAutomationOpenInput,
   type PreviewAutomationPressInput,
   type PreviewAutomationRequest,
   type PreviewAutomationResizeInput,
   type PreviewAutomationScrollInput,
+  type PreviewAutomationSelectInput,
   type PreviewAutomationSetColorSchemeInput,
   type PreviewAutomationStatus,
   type PreviewAutomationTypeInput,
@@ -1462,6 +1465,18 @@ const make = Effect.gen(function* () {
           .catch(constVoid);
         return undefined;
       }
+      case "hover":
+        return recordAction(tab, "hover", () =>
+          ServerBrowserPage.hover(tab.page, input as PreviewAutomationHoverInput),
+        );
+      case "select":
+        return recordAction(tab, "select", () =>
+          ServerBrowserPage.select(tab.page, input as PreviewAutomationSelectInput),
+        );
+      case "drag":
+        return recordAction(tab, "drag", () =>
+          ServerBrowserPage.drag(tab.page, input as PreviewAutomationDragInput),
+        );
       case "upload":
         return recordAction(tab, "upload", () =>
           uploadFiles(tab, input as PreviewAutomationUploadInput),

apps/server/src/preview/ServerBrowserPage.test.ts

@@ -179,4 +179,39 @@ describe("server browser element refs", () => {
     await ServerBrowserPage.click(page, { selector: 'button[data-example=" >> aria-ref=e1"]' });
     expect(await page.locator("button").textContent()).toBe("saved");
   });
+
+  it("right-clicks, double-clicks, hovers, selects, and drags like a pointer user", async () => {
+    await page.setContent(`
+      <style>#menu { position: absolute; display: none } #hover:hover + #menu { display: block }</style>
+      <button id="target">target</button>
+      <div id="hover">hover me</div><div id="menu">menu item</div>
+      <select id="size"><option value="s">Small</option><option value="l">Large</option></select>
+      <div id="card" draggable="true">card</div><div id="lane" style="height:40px">lane</div>
+      <p id="log"></p>
+      <script>
+        const log = (text) => (document.getElementById("log").textContent += text + ";");
+        const target = document.getElementById("target");
+        target.addEventListener("contextmenu", (event) => { event.preventDefault(); log("context"); });
+        target.addEventListener("dblclick", () => log("dblclick"));
+        document.getElementById("card").addEventListener("dragstart", (event) =>
+          event.dataTransfer.setData("text/plain", "card"),
+        );
+        for (const type of ["dragenter", "dragover"])
+          document.getElementById("lane").addEventListener(type, (event) => event.preventDefault());
+        document.getElementById("lane").addEventListener("drop", () => log("drop"));
+      </script>`);
+    await ServerBrowserPage.click(page, { locator: "#target", button: "right" });
+    await ServerBrowserPage.click(page, { locator: "#target", clickCount: 2 });
+    await ServerBrowserPage.hover(page, { locator: "#hover" });
+    expect(await page.isVisible("#menu")).toBe(true);
+    // A visible label selects the same option as its value.
+    expect(await ServerBrowserPage.select(page, { locator: "#size", values: ["Large"] })).toEqual({
+      selected: ["l"],
+    });
+    await ServerBrowserPage.drag(page, { source: "#card", target: "#lane" });
+    expect(await page.textContent("#log")).toBe("context;dblclick;drop;");
+    await expect(
+      ServerBrowserPage.select(page, { locator: "#target", values: ["s"] }),
+    ).rejects.toMatchObject({ tag: "PreviewAutomationTargetNotEditableError" });
+  });
 });

apps/server/src/preview/ServerBrowserPage.ts

@@ -4,10 +4,14 @@ import {
   PROVIDER_SEND_TURN_MAX_FILE_BYTES,
   type PreviewAutomationClickInput,
   type PreviewAutomationConsoleEntry,
+  type PreviewAutomationDragInput,
   type PreviewAutomationEvaluateInput,
+  type PreviewAutomationHoverInput,
   type PreviewAutomationNetworkEntry,
   type PreviewAutomationPressInput,
   type PreviewAutomationScrollInput,
+  type PreviewAutomationSelectInput,
+  type PreviewAutomationSelectResult,
   type PreviewAutomationSnapshot,
   type PreviewAutomationTypeInput,
   type PreviewAutomationUploadInput,
@@ -216,8 +220,11 @@ export const click = async (
     const box = await locator.boundingBox({ timeout });
     if (box) point = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
   }
+  const options = { button: input.button ?? "left", clickCount: input.clickCount ?? 1 } as const;
   const clicked =
-    locator === null ? page.mouse.click(point.x, point.y) : locator.click({ timeout });
+    locator === null
+      ? page.mouse.click(point.x, point.y, options)
+      : locator.click({ ...options, timeout });
   let onDialog = constVoid;
   const dialogOpened = new Promise<"dialog">((resolve) => {
     onDialog = () => resolve("dialog");
@@ -293,6 +300,58 @@ export const type = async (page: Page, input: PreviewAutomationTypeInput) => {
   await page.keyboard.insertText(input.text);
 };
 
+/** Hovering by locator moves the real mouse, so CSS :hover and pointer events both apply. */
+export const hover = async (page: Page, input: PreviewAutomationHoverInput) => {
+  const locator = targetLocator(page, input);
+  if (locator === null) {
+    await page.mouse.move(input.x ?? 0, input.y ?? 0);
+    return;
+  }
+  await locator.hover({ timeout: input.timeoutMs ?? DEFAULT_TIMEOUT_MS });
+};
+
+export const select = async (
+  page: Page,
+  input: PreviewAutomationSelectInput,
+): Promise<PreviewAutomationSelectResult> => {
+  const locator = targetLocator(page, input)!;
+  const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  // Each entry matches an option by value first, then by its visible label.
+  const options = await locator.evaluate(
+    (element) =>
+      element.tagName === "SELECT"
+        ? Array.from(
+            (element as unknown as { options: ArrayLike<{ value: string; label: string }> })
+              .options,
+            ({ value, label }) => ({ value, label: label.trim() }),
+          )
+        : null,
+    undefined,
+    { timeout },
+  );
+  const values =
+    options &&
+    input.values.map(
+      (entry) =>
+        (
+          options.find((option) => option.value === entry) ??
+          options.find((option) => option.label === entry.trim())
+        )?.value ?? entry,
+    );
+  if (values === null)
+    throw new ServerBrowserOperationError(
+      "PreviewAutomationTargetNotEditableError",
+      "This element is not a <select>. Click a custom dropdown, then click its option.",
+    );
+  return { selected: await locator.selectOption(values, { timeout }) };
+};
+
+export const drag = async (page: Page, input: PreviewAutomationDragInput) => {
+  const source = targetLocator(page, { locator: input.source })!;
+  const target = targetLocator(page, { locator: input.target })!;
+  await source.dragTo(target, { timeout: input.timeoutMs ?? DEFAULT_TIMEOUT_MS });
+};
+
 /** Sets files on one file input; false when no locator or selector names one. */
 export const setInputFiles = async (page: Page, input: PreviewAutomationUploadInput) => {
   const locator = targetLocator(page, input);